00000001:01000000:2.0F:1789760633.553054:0:2297:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm adler32 speed = 799 MB/s 00000001:01000000:3.0F:1789760633.803368:0:2297:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm crc32 speed = 1023 MB/s 00000001:01000000:1.0F:1789760634.052509:0:2297:0:(lnet-crypto.c:360:cfs_crypto_performance_test()) Crypto hash algorithm crc32c speed = 6699 MB/s 00000020:02000000:0.0F:1789760635.005168:0:2327:0:(class_obd.c:771:obdclass_init()) Lustre: Build Version: 2.17.57_103_g082f77f 00000400:02000000:3.0:1789760636.036383:0:2342:0:(api-ni.c:2823:lnet_startup_lndni()) Added LNI 192.168.203.13@tcp [8/256/0/180] 00008000:02000000:2.0:1789760639.663533:0:2492:0:(echo_client.c:2574:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:01000004:2.0:1789760757.078006:0:5431:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:2.0:1789760757.078012:0:5431:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:2.0:1789760757.078013:0:5431:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.113@tcp:/lustre 00000020:01000004:2.0:1789760757.078014:0:5431:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:2.0:1789760757.078164:0:5431:0:(super25.c:131:lustre_fill_super()) VFS Op: sb 00000000850223fb 00000080:01000004:2.0:1789760757.078175:0:5431:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:2.0:1789760757.078195:0:5431:0:(obd_mount.c:411:lustre_start_mgc()) Start MGC 'MGC192.168.203.113@tcp' 00000020:01000004:2.0:1789760757.078250:0:5431:0:(obd_mount.c:230:lustre_start_simple()) Starting OBD MGC192.168.203.113@tcp (typ=mgc) 00000020:00000080:2.0:1789760757.078276:0:5431:0:(genops.c:413:class_newdev()) Allocate new device MGC192.168.203.113@tcp (00000000116c05a2) 00000020:00000080:2.0:1789760757.078291:0:5431:0:(obd_config.c:720:class_attach_name()) OBD: dev 0 attached type mgc with refcount 2 00000100:00100000:2.0:1789760757.078726:0:5431:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1789760757.078753:0:5431:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00100000:2.0:1789760757.078927:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00100000:2.0:1789760757.078935:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00100000:2.0:1789760757.079378:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00100000:2.0:1789760757.079385:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00100000:3.0:1789760757.080389:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00100000:3.0:1789760757.080400:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000100:00100000:3.0:1789760757.081825:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00100000:2.0:1789760757.082332:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00100000:2.0:1789760757.083663:0:5431:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:2.0:1789760757.083980:0:5431:0:(service.c:155:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000100:00100000:2.0:1789760757.084206:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000100:00100000:2.0:1789760757.084211:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000100:00100000:2.0:1789760757.085851:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00100000:2.0:1789760757.085866:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00100000:3.0:1789760757.086927:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00100000:3.0:1789760757.086946:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000100:00100000:2.0:1789760757.087718:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00100000:2.0:1789760757.087727:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00100000:2.0:1789760757.088283:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00100000:2.0:1789760757.088289:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00100000:3.0:1789760757.088724:0:5431:0:(service.c:3462:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00100000:3.0:1789760757.088731:0:5431:0:(service.c:3521:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00010000:00010000:1.0:1789760757.089493:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760757.089497:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760757.089499:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760757.089790:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760757.089793:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760757.089794:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00080000:2.0:1789760757.090405:0:5431:0:(ldlm_lib.c:102:import_set_conn()) imp 000000000992d5da@MGC192.168.203.113@tcp: add connection 192.168.203.113@tcp at head 00010000:00010000:2.0:1789760757.090721:0:5431:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.203.113@tcp-0 is initialized 00000040:01000000:2.0:1789760757.090730:0:5431:0:(llog_obd.c:197:llog_setup()) obd MGC192.168.203.113@tcp ctxt 1 is initialized 10000000:01000000:2.0:1789760757.091283:0:5445:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 00000020:00000080:3.0:1789760757.091375:0:5431:0:(obd_config.c:826:class_setup()) finished setup of obd MGC192.168.203.113@tcp (uuid 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea) 00000020:00000080:3.0:1789760757.091402:0:5431:0:(genops.c:1385:class_connect()) connect: client 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea, cookie 0x3d1f44aaa2c561a7 00000100:00080000:3.0:1789760757.091420:0:5431:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:3.0:1789760757.091429:0:5431:0:(import.c:546:import_select_connection()) MGC192.168.203.113@tcp: skip NID 192.168.203.113@tcp as not ready: rc = -114 00000100:00080000:3.0:1789760757.091435:0:5431:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import 000000000992d5da using connection 192.168.203.113@tcp/192.168.203.113@tcp 00010000:00010000:2.0:1789760757.091539:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760757.091689:0:5431:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@00000000707126bd x1876700048654464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760757.091838:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048654464:192.168.203.113@tcp:250: 00000100:00080000:2.0:1789760757.091867:0:5431:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS 00000080:00200000:2.0:1789760757.091888:0:5431:0:(llite_lib.c:1506:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff9811e1a82800 (sb 00000000850223fb) 00000080:01000000:2.0:1789760757.091977:0:5431:0:(llite_lib.c:1232:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:2.0:1789760757.091981:0:5431:0:(llite_lib.c:1242:ll_options()) next opt=user_xattr 00000080:00000004:2.0:1789760757.091986:0:5431:0:(llite_lib.c:1242:ll_options()) next opt=flock 00000080:01000000:2.0:1789760757.091993:0:5431:0:(llite_lib.c:1540:ll_fill_super()) llite sb uuid: c2c87d9a-ec5f-46cf-8e98-545b26703d9f 10000000:01000000:2.0:1789760757.092682:0:5431:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:2.0:1789760757.092686:0:5431:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff9811e1a82800 10000000:01000000:2.0:1789760757.092692:0:5431:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffff9811c2e4b370 10000000:01000000:2.0:1789760757.092699:0:5431:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:2.0:1789760757.092708:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff9811c2e4b370 from 1 10000000:01000000:2.0:1789760757.092711:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:2.0:1789760757.092759:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760757.092771:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760757.092784:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:2.0:1789760757.092804:0:5431:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@00000000dc198017 x1876700048654592/t0(0) o101->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760757.092814:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d65e9c41 going to sleep for 0 seconds 00000100:00100000:3.0:1789760758.147641:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1789760757/real 0] req@00000000707126bd x1876700048654464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760758 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760758.147663:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@00000000707126bd x1876700048654464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760758 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760758.147669:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d65e9c41 going to sleep for 0 seconds 00000100:00080000:3.0:1789760758.147689:0:2358:0:(import.c:65:import_set_state_nolock()) 000000000992d5da MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760758.147728:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760758.147734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048654464:192.168.203.113@tcp:250: 00010000:00010000:2.0:1789760758.147746:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760758.147872:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 0 suppress 0 00000100:00080000:0.0:1789760758.147878:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760758.147882:0:488:0:(import.c:65:import_set_state_nolock()) 000000000992d5da MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760758.147891:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 211 00000100:00080000:0.0:1789760758.147908:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import 000000000992d5da using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760758.147974:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@0000000020121b9f x1876700048654720/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760758.148104:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@0000000020121b9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048654720:192.168.203.113@tcp:250: 00000100:00100000:3.0:1789760758.152347:0:2358:0:(client.c:3125:ptlrpc_free_committed()) MGC192.168.203.113@tcp: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760758.152365:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) MGC192.168.203.113@tcp: connect to target with instance 0 00000100:00080000:3.0:1789760758.152374:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) MGC192.168.203.113@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 10000000:01000000:3.0:1789760758.152380:0:2358:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808005 00000100:00080000:3.0:1789760758.152400:0:2358:0:(import.c:65:import_set_state_nolock()) 000000000992d5da MGS: changing import state from CONNECTING to FULL 10000000:01000000:3.0:1789760758.152404:0:2358:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808004 00000100:00080000:3.0:1789760758.152423:0:2358:0:(pinger.c:186:ptlrpc_pinger_ir_up()) IR up 00000100:00080000:3.0:1789760758.152434:0:2358:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000d65e9c41): req@00000000dc198017 x1876700048654592/t0(0) o101->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760758.152606:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@0000000020121b9f pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048654720:192.168.203.113@tcp:250: 00000100:00100000:0.0:1789760758.152915:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d65e9c41 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.155187:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000dc198017 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048654592:192.168.203.113@tcp:101: 00010000:00010000:0.0:1789760758.155226:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760758.155238:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760758.155309:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048654848:192.168.203.113@tcp:501: 00000100:00100000:0.0:1789760758.155380:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 0000000067b061f2 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.158967:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048654848:192.168.203.113@tcp:501: 10000000:01000000:0.0:1789760758.159009:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:0.0:1789760758.159033:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760758.159044:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: 0000000089df0efa/0x3d1f44aaa2c561ae lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1789760758.159052:0:5431:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff9811e1a82800 10000000:01000000:0.0:1789760758.159071:0:5431:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:0.0:1789760758.159075:0:5431:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff9811e1a82800 10000000:01000000:0.0:1789760758.159077:0:5431:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:0.0:1789760758.159080:0:5431:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff9811e1a82800 10000000:01000000:0.0:1789760758.159096:0:5431:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:0.0:1789760758.159100:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff9811e1a82800 from 1 10000000:01000000:0.0:1789760758.159102:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1789760758.159134:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760758.159141:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760758.159151:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760758.159170:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048654976:192.168.203.113@tcp:101: 00000100:00100000:0.0:1789760758.159248:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 0000000067b061f2 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.165703:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048654976:192.168.203.113@tcp:101: 00010000:00010000:0.0:1789760758.165731:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760758.165748:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760758.165821:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655104:192.168.203.113@tcp:501: 00000100:00100000:0.0:1789760758.165916:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 0000000067b061f2 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.172095:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655104:192.168.203.113@tcp:501: 00000100:00100000:0.0:1789760758.172170:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655232:192.168.203.113@tcp:503: 00000100:00100000:0.0:1789760758.172372:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 0000000067b061f2 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.177378:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655232:192.168.203.113@tcp:503: 00000100:00100000:0.0:1789760758.177905:0:5449:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5449:1876700048655360:192.168.203.113@tcp:502: 00000100:00100000:0.0:1789760758.178093:0:5449:0:(client.c:2715:ptlrpc_set_wait()) set 00000000493ce426 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.191809:0:5449:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@000000004d7366ff pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5449:1876700048655360:192.168.203.113@tcp:502: 00000020:01000000:0.0:1789760758.191854:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760758.191868:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.191870:0:5449:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:0.0:1789760758.191877:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffff9811e1a82800 00000020:00000080:0.0:1789760758.191881:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760758.191905:0:5449:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffff9811e1a82800 (0000000052d707fd) 00000020:00000080:0.0:1789760758.191926:0:5449:0:(obd_config.c:720:class_attach_name()) OBD: dev 1 attached type lov with refcount 2 00000020:01000000:0.0:1789760758.191933:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffff9811e1a82800 00000020:00000080:0.0:1789760758.191935:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:0.0:1789760758.192322:0:5449:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffff9811e1a82800 (uuid c2c87d9a-ec5f-46cf-8e98-545b26703d9f) 00000020:01000000:0.0:1789760758.192327:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.192331:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.192332:0:5449:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:0.0:1789760758.192336:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760758.192339:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.192340:0:5449:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:0.0:1789760758.192344:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffff9811e1a82800 00000020:00000080:0.0:1789760758.192346:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760758.192352:0:5449:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffff9811e1a82800 (0000000010ac1874) 00000020:00000080:0.0:1789760758.192362:0:5449:0:(obd_config.c:720:class_attach_name()) OBD: dev 2 attached type lmv with refcount 2 00000020:01000000:0.0:1789760758.192366:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffff9811e1a82800 00000020:00000080:0.0:1789760758.192376:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:0.0:1789760758.193231:0:5449:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffff9811e1a82800 (uuid c2c87d9a-ec5f-46cf-8e98-545b26703d9f) 00000020:01000000:0.0:1789760758.193238:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.193241:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.193243:0:5449:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:0.0:1789760758.193245:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760758.193249:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.193250:0:5449:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:0.0:1789760758.193254:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:0.0:1789760758.193255:0:5449:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:0.0:1789760758.193258:0:5449:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:0.0:1789760758.193263:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.193265:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.193267:0:5449:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:0.0:1789760758.193269:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760758.193273:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.193274:0:5449:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:0.0:1789760758.193277:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760758.193282:0:5449:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760758.193306:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.193308:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760758.193316:0:5449:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffff9811e1a82800 (000000000a82431f) 00000020:00000080:0.0:1789760758.193325:0:5449:0:(obd_config.c:720:class_attach_name()) OBD: dev 3 attached type mdc with refcount 2 00000020:01000000:0.0:1789760758.193329:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.193331:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760758.193391:0:5449:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000b4f72292@lustre-MDT0000-mdc-ffff9811e1a82800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760758.197400:0:5449:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffff9811e1a82800-1 is initialized 00000040:01000000:0.0:1789760758.198858:0:5449:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffff9811e1a82800 ctxt 13 is initialized 00000020:00000080:0.0:1789760758.200209:0:5449:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffff9811e1a82800 (uuid c2c87d9a-ec5f-46cf-8e98-545b26703d9f) 00000020:01000000:0.0:1789760758.200226:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff9811e1a82800 00000020:00000080:0.0:1789760758.200231:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:0.0:1789760758.200261:0:5449:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:0.0:1789760758.200297:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.200301:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.200303:0:5449:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:0.0:1789760758.200306:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1789760758.200308:0:5449:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #8 00000020:01000000:0.0:1789760758.200310:0:5449:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760758.200311:0:5449:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760758.200313:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1789760758.200315:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.200316:0:5449:0:(obd_config.c:1563:class_process_config()) marker 8 (0x6) lustre sys.timeout 00000020:01000000:0.0:1789760758.200319:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1789760758.200323:0:5449:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.113@tcp:/lustre 00000020:00000080:0.0:1789760758.200326:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.200327:0:5449:0:(obd_config.c:1563:class_process_config()) marker 13 (0x1) lustre-OST0000 add osc 00000020:00000080:0.0:1789760758.200330:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760758.200335:0:5449:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760758.200361:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.200364:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760758.200374:0:5449:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffff9811e1a82800 (00000000c134e4f2) 00000020:00000080:0.0:1789760758.200388:0:5449:0:(obd_config.c:720:class_attach_name()) OBD: dev 4 attached type osc with refcount 2 00000020:01000000:0.0:1789760758.200393:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.200396:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760758.200825:0:5449:0:(ldlm_lib.c:102:import_set_conn()) imp 00000000f69c02b8@lustre-OST0000-osc-ffff9811e1a82800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760758.205835:0:5449:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffff9811e1a82800-2 is initialized 00000020:00000080:0.0:1789760758.211317:0:5449:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffff9811e1a82800 (uuid c2c87d9a-ec5f-46cf-8e98-545b26703d9f) 00000020:01000000:0.0:1789760758.211335:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9811e1a82800 00000020:00000080:0.0:1789760758.211340:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1789760758.211362:0:5449:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:0.0:1789760758.211371:0:5449:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9811e1a82800: Adding target lustre-OST0000_UUID 00020000:01000000:0.0:1789760758.211378:0:5449:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:0.0:1789760758.211390:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.211401:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.211403:0:5449:0:(obd_config.c:1563:class_process_config()) marker 13 (0x2) lustre-OST0000 add osc 00000020:01000000:0.0:1789760758.211407:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1789760758.211409:0:5449:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #15 00000020:01000000:0.0:1789760758.211411:0:5449:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760758.211413:0:5449:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760758.211414:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1789760758.211416:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.211418:0:5449:0:(obd_config.c:1563:class_process_config()) marker 15 (0x6) lustre sys.timeout 00000020:01000000:0.0:1789760758.211420:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1789760758.211424:0:5449:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.113@tcp:/lustre 00000020:00000080:0.0:1789760758.211427:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.211428:0:5449:0:(obd_config.c:1563:class_process_config()) marker 19 (0x1) lustre-OST0001 add osc 00000020:00000080:0.0:1789760758.211431:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760758.211437:0:5449:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760758.216323:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.216331:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760758.216352:0:5449:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffff9811e1a82800 (00000000dd9fa0d5) 00000020:00000080:0.0:1789760758.216372:0:5449:0:(obd_config.c:720:class_attach_name()) OBD: dev 5 attached type osc with refcount 2 00000020:01000000:0.0:1789760758.216380:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffff9811e1a82800 00000020:00000080:0.0:1789760758.216384:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760758.216516:0:5449:0:(ldlm_lib.c:102:import_set_conn()) imp 000000000b4f29b7@lustre-OST0001-osc-ffff9811e1a82800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760758.216793:0:5449:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffff9811e1a82800-3 is initialized 00000020:00000080:0.0:1789760758.217115:0:5449:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffff9811e1a82800 (uuid c2c87d9a-ec5f-46cf-8e98-545b26703d9f) 00000020:01000000:0.0:1789760758.217197:0:5449:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9811e1a82800 00000020:00000080:0.0:1789760758.217200:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1789760758.217216:0:5449:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:0.0:1789760758.217224:0:5449:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9811e1a82800: Adding target lustre-OST0001_UUID 00020000:01000000:0.0:1789760758.217228:0:5449:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:0.0:1789760758.217247:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.217250:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.217253:0:5449:0:(obd_config.c:1563:class_process_config()) marker 19 (0x2) lustre-OST0001 add osc 00000020:01000000:0.0:1789760758.217264:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760758.217268:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.217269:0:5449:0:(obd_config.c:1563:class_process_config()) marker 21 (0x1) lustre sys.timeout 00000020:00000080:0.0:1789760758.217273:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:0.0:1789760758.217274:0:5449:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 100 to 20 00000100:00100000:0.0:1789760758.217312:0:5449:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000acbbccc6 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5449:1876700048655488:192.168.203.113@tcp:502: 00000100:00100000:0.0:1789760758.217412:0:5449:0:(client.c:2715:ptlrpc_set_wait()) set 00000000493ce426 going to sleep for 11 seconds 00000100:00100000:0.0:1789760758.222432:0:5449:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000acbbccc6 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5449:1876700048655488:192.168.203.113@tcp:502: 00000020:01000000:0.0:1789760758.222513:0:5449:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760758.222521:0:5449:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760758.222524:0:5449:0:(obd_config.c:1563:class_process_config()) marker 21 (0x2) lustre sys.timeout 00000040:00080000:0.0:1789760758.222527:0:5449:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 39 count 39 00000020:01000000:3.0:1789760758.222722:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-38 (rc=0) 10000000:01000000:3.0:1789760758.222739:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:3.0:1789760758.222762:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.222773:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: 00000000caa0c1c2/0x3d1f44aaa2c561b5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760758.222781:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9811e1a82800 from 1 10000000:01000000:3.0:1789760758.222783:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:3.0:1789760758.222819:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.222827:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.222836:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760758.222856:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655616:192.168.203.113@tcp:101: 00000100:00100000:3.0:1789760758.222933:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d2601322 going to sleep for 11 seconds 00000100:00100000:3.0:1789760758.225067:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655616:192.168.203.113@tcp:101: 00010000:00010000:3.0:1789760758.225093:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d9 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.225117:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c528d9 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760758.225687:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655744:192.168.203.113@tcp:256: 00000100:00100000:3.0:1789760758.225781:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d2601322 going to sleep for 11 seconds 00000100:00100000:3.0:1789760758.229023:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655744:192.168.203.113@tcp:256: 10000000:01000000:3.0:1789760758.229661:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:3.0:1789760758.229678:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d9 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.229689:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: 00000000f3cc520b/0x3d1f44aaa2c561df lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528d9 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760758.229698:0:5431:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9811e1a82800 from 1 10000000:01000000:3.0:1789760758.229700:0:5431:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:3.0:1789760758.229732:0:5431:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: 0000000066f7d235/0x3d1f44aaa2c561e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.229739:0:5431:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: 0000000066f7d235/0x3d1f44aaa2c561e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.229748:0:5431:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: 0000000066f7d235/0x3d1f44aaa2c561e6 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760758.229767:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655872:192.168.203.113@tcp:101: 00000100:00100000:3.0:1789760758.229839:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000d2601322 going to sleep for 11 seconds 00000100:00100000:1.0:1789760758.235505:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@00000000707126bd pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048655872:192.168.203.113@tcp:101: 00010000:00010000:1.0:1789760758.235532:0:5431:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: 0000000066f7d235/0x3d1f44aaa2c561e6 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760758.235543:0:5431:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: 0000000066f7d235/0x3d1f44aaa2c561e6 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760758.235635:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048656000:192.168.203.113@tcp:501: 00000100:00100000:1.0:1789760758.235705:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000534f6642 going to sleep for 11 seconds 00000100:00100000:1.0:1789760758.241191:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048656000:192.168.203.113@tcp:501: 00000100:00100000:1.0:1789760758.241250:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048656128:192.168.203.113@tcp:503: 00000100:00100000:1.0:1789760758.241345:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set 00000000534f6642 going to sleep for 11 seconds 00000100:00100000:1.0:1789760758.243362:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5431:1876700048656128:192.168.203.113@tcp:503: 00000100:00100000:1.0:1789760758.244225:0:5454:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5454:1876700048656256:192.168.203.113@tcp:502: 00000100:00100000:1.0:1789760758.244316:0:5454:0:(client.c:2715:ptlrpc_set_wait()) set 0000000017543dc7 going to sleep for 11 seconds 00000100:00100000:1.0:1789760758.248141:0:5454:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@0000000074ea8ea7 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5454:1876700048656256:192.168.203.113@tcp:502: 00000020:01000000:1.0:1789760758.248180:0:5454:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1789760758.248191:0:5454:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760758.248194:0:5454:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:1.0:1789760758.248200:0:5454:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9811e1a82800 00000020:00000080:1.0:1789760758.248202:0:5454:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:1.0:1789760758.248299:0:5454:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5454:1876700048656384:192.168.203.113@tcp:502: 00000100:00100000:1.0:1789760758.248390:0:5454:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d82ca600 going to sleep for 11 seconds 00000100:00100000:1.0:1789760758.251080:0:5454:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5454:1876700048656384:192.168.203.113@tcp:502: 00000020:01000000:1.0:1789760758.251127:0:5454:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760758.251136:0:5454:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760758.251139:0:5454:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000040:00080000:1.0:1789760758.251143:0:5454:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 4 count 4 00000020:01000000:3.0:1789760758.251297:0:5431:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-3 (rc=0) 10000000:01000000:3.0:1789760758.251327:0:5431:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1789760758.251349:0:5431:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760758.251361:0:5431:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:3.0:1789760758.251384:0:5431:0:(llite_lib.c:1614:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:3.0:1789760758.251408:0:5431:0:(genops.c:1385:class_connect()) connect: client c2c87d9a-ec5f-46cf-8e98-545b26703d9f, cookie 0x3d1f44aaa2c561ed 00800000:01000000:3.0:1789760758.251433:0:5431:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect c2c87d9a-ec5f-46cf-8e98-545b26703d9f to lustre-clilmv-ffff9811e1a82800 00800000:01000000:3.0:1789760758.251440:0:5431:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffff9811e1a82800(c2c87d9a-ec5f-46cf-8e98-545b26703d9f) - lustre-MDT0000_UUID, c2c87d9a-ec5f-46cf-8e98-545b26703d9f 00000020:00000080:3.0:1789760758.251489:0:5431:0:(genops.c:1385:class_connect()) connect: client c2c87d9a-ec5f-46cf-8e98-545b26703d9f, cookie 0x3d1f44aaa2c561f4 00000100:00080000:3.0:1789760758.251498:0:5431:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:3.0:1789760758.251506:0:5431:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760758.251518:0:5431:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1789760758.251542:0:5431:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048656512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760758.251565:0:5431:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID 00800000:01000000:3.0:1789760758.251651:0:5431:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffff9811e1a82800(c2c87d9a-ec5f-46cf-8e98-545b26703d9f) successfully (5) 00800000:00000004:3.0:1789760758.251655:0:5431:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffff9811e1a82800: age -788, max_age 0 00000080:00000004:3.0:1789760758.251686:0:5431:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffff9811e1a82800: age -788, max_age 211 00800000:00000004:3.0:1789760758.251692:0:5431:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff9811e1a82800: age -788, max_age 211 00000100:00080000:3.0:1789760758.251710:0:5431:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9811e1a20e00 x1876700048656768/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760758.251721:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d80ab840 going to sleep for 0 seconds 00000100:00080000:3.0:1789760758.251781:0:2362:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9811e1a21f80 x1876700048656640/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760758.251981:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048656512:192.168.203.113@tcp:38: 00010000:00080000:2.0:1789760758.258987:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a20380 x1876700048656512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/416 e 0 to 0 dl 1789760768 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760758.259003:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9811e1a82800: committing for last_committed 0 gen 1 00000100:00080000:2.0:1789760758.259012:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to target with instance 1 00000001:01000000:3.0:1789760758.509098:0:2358:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9811e1a82800: T10 checksum algorithm t10ip512 speed = 2943 MB/s 00000001:01000000:3.0:1789760758.759173:0:2358:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9811e1a82800: T10 checksum algorithm t10ip4K speed = 3891 MB/s 00000001:01000000:2.0:1789760759.011137:0:2358:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9811e1a82800: T10 checksum algorithm t10crc512 speed = 757 MB/s 00010000:00010000:3.0:1789760759.173433:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:01000000:0.0:1789760759.260721:0:2358:0:(integrity.c:236:obd_t10_performance_test()) lustre-MDT0000-mdc-ffff9811e1a82800: T10 checksum algorithm t10crc4K speed = 823 MB/s 00000100:00080000:0.0:1789760759.260734:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffff9811e1a82800: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:0.0:1789760759.260749:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:0.0:1789760759.260755:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:3.0:1789760759.268709:0:2358:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000059b80e7c): req@ffff9811e1a20e00 x1876700048656768/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760759.268746:0:2358:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000d3a38b88): req@ffff9811e1a21f80 x1876700048656640/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760759.268771:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048656512:192.168.203.113@tcp:38: 00000100:00100000:1.0:1789760759.268840:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d80ab840 going to sleep for 0 seconds 00000100:00100000:1.0:1789760759.268961:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d80ab840 going to sleep for 11 seconds 00010000:00080000:2.0:1789760759.275421:0:2362:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a21f80 x1876700048656640/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 1789760775 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760759.275455:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:0:1876700048656640:192.168.203.113@tcp:41: 00010000:00080000:3.0:1789760759.275720:0:5431:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a20e00 x1876700048656768/t0(0) o41->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 1789760775 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760759.275731:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5431:1876700048656768:192.168.203.113@tcp:41: 00800000:00000004:3.0:1789760759.275747:0:5431:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffff9811e1a82800: update ffff9811c6dfb7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00000004:3.0:1789760759.275754:0:5431:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffff9811e1a82800: update ffff9811c6df9e08 cache blocks 317407/353529 objects 1023725/1024000 00000100:00100000:3.0:1789760759.275792:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5431:1876700048656896:192.168.203.113@tcp:40: 00000100:00100000:3.0:1789760759.275907:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d80ab840 going to sleep for 11 seconds 00010000:00080000:3.0:1789760759.288955:0:5431:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a20e00 x1876700048656896/t0(0) o40->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 440/448 e 0 to 0 dl 1789760775 ref 2 fl Rpc:RQU/209/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760759.288977:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5431:1876700048656896:192.168.203.113@tcp:40: 00000080:00000004:3.0:1789760759.289001:0:5431:0:(llite_lib.c:614:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:3.0:1789760759.289015:0:5431:0:(llite_lib.c:775:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 2146304 00020000:01000000:3.0:1789760759.289091:0:5431:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:3.0:1789760759.289106:0:5431:0:(genops.c:1385:class_connect()) connect: client c2c87d9a-ec5f-46cf-8e98-545b26703d9f, cookie 0x3d1f44aaa2c561fb 00000020:00000080:3.0:1789760759.289137:0:5431:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x3d1f44aaa2c56202 00000100:00080000:3.0:1789760759.289146:0:5431:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:3.0:1789760759.289153:0:5431:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e1a82800: import ffff9811e071f800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760759.289165:0:5431:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1789760759.289189:0:5431:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048657024/t0(0) o8->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760759.289268:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048657024:192.168.203.113@tcp:8: 00000100:00080000:2.0:1789760759.290703:0:5431:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-OST0000_UUID 00020000:01000000:2.0:1789760759.290731:0:5431:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffff9811e1a82800) active 00000020:00000080:2.0:1789760759.290775:0:5431:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x3d1f44aaa2c56209 00000100:00080000:2.0:1789760759.290786:0:5431:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:2.0:1789760759.290795:0:5431:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e1a82800: import ffff9811e0719800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760759.290806:0:5431:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760759.290830:0:5431:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048657152/t0(0) o8->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760759.291008:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048657280:192.168.203.113@tcp:400: 00000100:00080000:2.0:1789760759.292835:0:5431:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-OST0001_UUID 00020000:01000000:2.0:1789760759.292905:0:5431:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffff9811e1a82800) active 00000100:00100000:2.0:1789760759.292980:0:5431:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5431:1876700048657408:192.168.203.113@tcp:33: 00000100:00100000:2.0:1789760759.293148:0:5431:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e0b1b540 going to sleep for 11 seconds 00000100:00100000:0.0:1789760759.293276:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048657152:192.168.203.113@tcp:8: 00010000:00080000:3.0:1789760759.294413:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a20e00 x1876700048657024/t0(0) o8->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760769 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760759.294427:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9811e1a82800: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760759.294450:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff9811e1a82800: connect to target with instance 1 00000100:00080000:3.0:1789760759.294491:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff9811e1a82800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1789760759.294505:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:3.0:1789760759.294508:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071f800 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1789760759.294515:0:2358:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:3.0:1789760759.294520:0:2358:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1789760759.294524:0:2358:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1789760759.294535:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048657024:192.168.203.113@tcp:8: 00000100:00100000:1.0:1789760759.305759:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048657280:192.168.203.113@tcp:400: 00010000:00080000:3.0:1789760759.305759:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d863bb80 x1876700048657152/t0(0) o8->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760769 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760759.305778:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9811e1a82800: committing for last_committed 0 gen 1 00000100:00080000:3.0:1789760759.305787:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff9811e1a82800: connect to target with instance 1 00000100:00080000:3.0:1789760759.305797:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff9811e1a82800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:3.0:1789760759.305809:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:3.0:1789760759.305814:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0719800 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:3.0:1789760759.305821:0:2358:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:3.0:1789760759.305839:0:2358:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:3.0:1789760759.305844:0:2358:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:3.0:1789760759.305854:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048657152:192.168.203.113@tcp:8: 00010000:00080000:2.0:1789760759.331840:0:5431:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d8639180 x1876700048657408/t0(0) o33->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:23/10 lens 448/464 e 0 to 0 dl 1789760775 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000100:00100000:2.0:1789760759.331860:0:5431:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5431:1876700048657408:192.168.203.113@tcp:33: 00000080:00200000:2.0:1789760759.332092:0:5431:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00200000:2.0:1789760759.332101:0:5431:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:02000400:2.0:1789760759.332170:0:5431:0:(llite_lib.c:1653:ll_fill_super()) Mounted lustre-client - version 2.17.57_103_g082f77f 00000080:00000004:2.0:1789760759.340171:0:5431:0:(super25.c:182:lustre_fill_super()) 192.168.203.113@tcp:/lustre: Mount complete 00000080:00200008:3.0:1789760759.492400:0:5471:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1789760759.492412:0:5471:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:3.0:1789760759.492669:0:5471:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760759.492734:0:5471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 5471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.492745:0:5471:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5471 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.492754:0:5471:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 5471 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760759.492778:0:5471:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job mkdir:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5471:1876700048657536:192.168.203.113@tcp:101: 00000100:00100000:3.0:1789760759.492907:0:5471:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d80ab840 going to sleep for 11 seconds 00000100:00100000:3.0:1789760759.496982:0:5471:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job mkdir:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5471:1876700048657536:192.168.203.113@tcp:101: 00010000:00010000:3.0:1789760759.497017:0:5471:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497029:0:5471:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497041:0:5471:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497056:0:5471:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497082:0:5471:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497090:0:5471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497098:0:5471:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1789760759.497109:0:5471:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x8 disposition=0xb, status=0 req@ffff9811e1a20e00 x1876700048657536/t0(0) o101->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 576/688 e 0 to 0 dl 1789760775 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:3.0:1789760759.497123:0:5471:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497136:0:5471:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff9811c7dd5c00 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1789760759.497142:0:5471:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1789760759.497165:0:5471:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00000080:00010000:3.0:1789760759.497169:0:5471:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:3.0:1789760759.497174:0:5471:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:3.0:1789760759.497177:0:5471:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e43c2fcb8 00010000:00010000:3.0:1789760759.497180:0:5471:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760759.497189:0:5471:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789760760.195648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760761.221508:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1789760761.376837:0:5523:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760761.376851:0:5523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:1.0:1789760761.376876:0:5523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760761.376895:0:5523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1789760761.376910:0:5523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:1.0:1789760761.376919:0:5523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760761.376927:0:5523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:1.0:1789760761.376931:0:5523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:1.0:1789760761.376938:0:5523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e43e27cb8 00010000:00010000:1.0:1789760761.376942:0:5523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:1.0:1789760761.376963:0:5523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:1.0:1789760761.381242:0:5523:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 0, flags 0, foreign 0 00000080:00200000:1.0:1789760761.381256:0:5523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:1.0:1789760761.381277:0:5523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760761.381296:0:5523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1789760761.381312:0:5523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:1.0:1789760761.381320:0:5523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760761.381327:0:5523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:1.0:1789760761.381331:0:5523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:1.0:1789760761.382931:0:5523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e43e27cb8 00010000:00010000:1.0:1789760761.382941:0:5523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:1.0:1789760761.382954:0:5523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1789760761.383140:0:5523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:1.0:1789760761.383157:0:5523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760761.383171:0:5523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1789760761.383194:0:5523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:1.0:1789760761.383202:0:5523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760761.383208:0:5523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:1.0:1789760761.383211:0:5523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:1.0:1789760761.383216:0:5523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e43e27bb0 00010000:00010000:1.0:1789760761.383220:0:5523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:1.0:1789760761.383228:0:5523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1789760761.383237:0:5523:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908), inode mode 41ed mask 44 00000080:00200008:1.0:1789760761.383246:0:5523:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), flags 100000 00000080:00200080:1.0:1789760761.385523:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.385546:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00800000:00000004:1.0:1789760761.385553:0:5523:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff9811e1a82800: age 213, max_age 215 00000100:00100000:1.0:1789760761.385593:0:5523:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657664:192.168.203.113@tcp:41: 00000100:00100000:1.0:1789760761.385680:0:5523:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d82cae40 going to sleep for 11 seconds 00000100:00100000:1.0:1789760761.389207:0:5523:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657664:192.168.203.113@tcp:41: 00800000:00000004:1.0:1789760761.389237:0:5523:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffff9811e1a82800: update ffff9811c6dfb7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00200080:1.0:1789760761.389309:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389326:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389399:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389415:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389433:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389439:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389444:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389448:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389452:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389455:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389459:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389493:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389499:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389503:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389507:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389512:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389517:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389521:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389525:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389529:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389534:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389538:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389542:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389546:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389550:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389554:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389558:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389562:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389576:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389581:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389585:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389589:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389593:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389597:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389601:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389605:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389610:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389613:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389618:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389622:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389626:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389631:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389636:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389641:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389646:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389650:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389655:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389659:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389665:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389669:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389673:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389676:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389680:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389690:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389695:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389700:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389705:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389709:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389714:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389718:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389722:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389725:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389730:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389735:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389739:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389743:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389748:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389753:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389757:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389761:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389767:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389783:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389789:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389794:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389798:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389801:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389806:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389809:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389822:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389827:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389832:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389835:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389840:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389843:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389848:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389851:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389856:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389859:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389863:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389866:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389870:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389873:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389876:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389879:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389883:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389886:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389890:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389893:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389897:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389901:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389905:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389909:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389914:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389923:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389929:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389933:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389938:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389943:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389948:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389951:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389956:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389959:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389963:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389968:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389972:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389977:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389980:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389984:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389987:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389990:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.389994:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.389998:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.390001:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.390005:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.390009:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.390012:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.390015:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00800000:00000080:1.0:1789760761.390019:0:5523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.390031:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.390038:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00020000:00000004:1.0:1789760761.390044:0:5523:0:(obd_class.h:1090:obd_statfs()) lustre-OST0000-osc-ffff9811e1a82800: age -788, max_age 215 00000100:00100000:1.0:1789760761.390083:0:5523:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657792:192.168.203.113@tcp:13: 00000100:00100000:1.0:1789760761.390158:0:5523:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d82cae40 going to sleep for 11 seconds 00010000:00080000:1.0:1789760761.403734:0:5523:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c7b80 x1876700048657792/t0(0) o13->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:7/4 lens 224/368 e 0 to 0 dl 1789760777 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760761.403752:0:5523:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657792:192.168.203.113@tcp:13: 00020000:00000004:1.0:1789760761.403793:0:5523:0:(obd_class.h:1116:obd_statfs()) lustre-OST0000-osc-ffff9811e1a82800: update ffff9811c6df8450 cache blocks 901403/958279 objects 261841/262144 00000080:00200080:1.0:1789760761.403863:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.403880:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00020000:00000004:1.0:1789760761.403887:0:5523:0:(obd_class.h:1090:obd_statfs()) lustre-OST0001-osc-ffff9811e1a82800: age -788, max_age 215 00000100:00100000:1.0:1789760761.403918:0:5523:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657920:192.168.203.113@tcp:13: 00000100:00100000:1.0:1789760761.403989:0:5523:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d82cae40 going to sleep for 11 seconds 00010000:00080000:1.0:1789760761.413758:0:5523:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c7b80 x1876700048657920/t0(0) o13->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:7/4 lens 224/368 e 0 to 0 dl 1789760777 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760761.413779:0:5523:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job lfs:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:5523:1876700048657920:192.168.203.113@tcp:13: 00020000:00000004:1.0:1789760761.413801:0:5523:0:(obd_class.h:1116:obd_statfs()) lustre-OST0001-osc-ffff9811e1a82800: update ffff9811c2cf8450 cache blocks 901403/958279 objects 261841/262144 00000080:00200080:1.0:1789760761.413867:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413883:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413889:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413893:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413897:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413900:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413918:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413922:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413925:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413929:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413932:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413935:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413938:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413941:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413945:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413947:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413951:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413953:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413957:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413959:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413963:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413966:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413969:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413972:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413975:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413978:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413981:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413984:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413987:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.413990:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.413993:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414000:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414004:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414008:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414011:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414014:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414017:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414020:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414023:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414026:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414030:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414032:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414036:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414038:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414042:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414045:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414048:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414051:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414054:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414057:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414060:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414063:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414066:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414069:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414072:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414075:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414084:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414089:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414093:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414096:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414099:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414102:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414106:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414109:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414113:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414116:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414121:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414124:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414128:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414132:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414136:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414141:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414145:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414148:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414152:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414155:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414159:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414162:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414166:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414169:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414173:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414190:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414196:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414199:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414204:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414207:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414212:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414215:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414219:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414222:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414227:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414230:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414234:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414237:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414241:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414244:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414247:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414251:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414254:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414258:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414262:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414265:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414269:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414272:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414276:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414280:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414291:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414295:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414299:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414302:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414306:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414310:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414313:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414317:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414321:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414324:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414328:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414332:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414335:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414339:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414343:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414347:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414350:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414354:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200080:1.0:1789760761.414357:0:5523:0:(dir.c:2004:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908) cmd=c00866a4 arg=7fff1459a5e0 00020000:00000080:1.0:1789760761.414360:0:5523:0:(lov_obd.c:976:lov_iocontrol()) lustre-clilov-ffff9811e1a82800: cmd=c00866a4 len=592 karg=ffff9811e06fa000 uarg=0000000000000000 00000080:00200008:1.0:1789760761.414373:0:5523:0:(file.c:479:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), flags 100000 00010000:00010000:2.0:1789760762.243760:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760763.267758:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760763.419179:0:5713:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:2.0:1789760764.293892:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760764.419764:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048658176:192.168.203.113@tcp:400: 00000100:00100000:3.0:1789760764.419764:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658048:192.168.203.113@tcp:400: 00000100:00100000:2.0:1789760764.419896:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048658432:192.168.203.113@tcp:400: 00000100:00100000:3.0:1789760764.419896:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048658304:192.168.203.113@tcp:400: 00000100:00100000:2.0:1789760764.423764:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048658176:192.168.203.113@tcp:400: 00000100:00100000:3.0:1789760764.423973:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658048:192.168.203.113@tcp:400: 00010000:00080000:2.0:1789760764.430419:0:2362:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d863ad80 x1876700048658432/t0(0) o400->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760780 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760764.430444:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048658432:192.168.203.113@tcp:400: 00010000:00080000:3.0:1789760764.430648:0:2361:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d8639c00 x1876700048658304/t0(0) o400->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760780 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760764.430661:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048658304:192.168.203.113@tcp:400: 00010000:00010000:3.0:1789760765.315919:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760766.339685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760766.620153:0:2352:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.113@tcp (source: 12345-192.168.203.113@tcp) 00000100:00100000:0.0:1789760766.620269:0:2352:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.113@tcp (source: 12345-192.168.203.113@tcp) 00000100:00100000:1.0:1789760766.620402:0:5432:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700066505472 00000100:00100000:1.0:1789760766.620415:0:5432:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700066505472 00000100:00100000:0.0:1789760766.620552:0:5433:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700066505600 00000100:00100000:0.0:1789760766.620557:0:5433:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700066505600 00000100:00100000:0.0:1789760766.620597:0:5433:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.113@tcp, seq: 0 00000100:00100000:0.0:1789760766.620607:0:5433:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9811e075df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+4:8814:x1876700066505600:12345-192.168.203.113@tcp:104: 00010000:00010000:0.0:1789760766.620630:0:5433:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760766.622178:0:5433:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1789760766.622205:0:5433:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9811e075df80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+4:8814:x1876700066505600:12345-192.168.203.113@tcp:104: Request processed in 1588us (1924us total) trans 0 rc 0/0 00000100:00100000:0.0:1789760766.622217:0:5433:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.113@tcp, seq: 0 00000100:00100000:1.0:1789760766.622584:0:5432:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.113@tcp, seq: 1 00000100:00100000:1.0:1789760766.622599:0:5432:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9811e075c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+3:8814:x1876700066505472:12345-192.168.203.113@tcp:104: 00010000:00010000:1.0:1789760766.622634:0:5432:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623552:0:5442:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:3.0:1789760766.623565:0:5442:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623576:0:5442:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9811c1391a00 already unused, calling callback (ffffffffc13dac20) 10000000:00010000:3.0:1789760766.623580:0:5442:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623589:0:5442:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1789760766.623596:0:5442:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760766.623603:0:5442:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1789760766.623728:0:5442:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623734:0:5442:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760766.623768:0:5442:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623775:0:5442:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.113@tcp lock: ffff9811c1391a00/0x3d1f44aaa2c561ae lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528cb expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623789:0:5442:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1789760766.623792:0:5442:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623797:0:5442:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9811c0031000 already unused, calling callback (ffffffffc13dac20) 10000000:00010000:3.0:1789760766.623800:0:5442:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.623806:0:5442:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1789760766.623813:0:5442:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760766.623818:0:5442:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:1.0:1789760766.623859:0:5432:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/0 locks to priority blp list, 0 blwis in pool 00000100:00100000:1.0:1789760766.623883:0:5432:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9811e075c380 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+3:8814:x1876700066505472:12345-192.168.203.113@tcp:104: Request processed in 1287us (3741us total) trans 0 rc 0/0 00000100:00100000:1.0:1789760766.623892:0:5432:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.113@tcp, seq: 1 00000100:00100000:2.0:1789760766.624076:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658560:192.168.203.113@tcp:103: 00010000:00010000:3.0:1789760766.624158:0:5442:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.624165:0:5442:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760766.624172:0:5442:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.624193:0:5442:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.113@tcp lock: ffff9811c0031000/0x3d1f44aaa2c561b5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528d2 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760766.624208:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760766.624210:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760766.624211:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760766.624229:0:2361:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760766.624235:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658688:192.168.203.113@tcp:103: 00010000:00010000:3.0:1789760766.624243:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760766.627113:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658688:192.168.203.113@tcp:103: 00000100:00100000:2.0:1789760766.627131:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048658560:192.168.203.113@tcp:103: 00010000:00010000:3.0:1789760767.363811:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:0.0:1789760768.004946:0:5445:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:0.0:1789760768.004954:0:5445:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff9811e1a82800 from 39 10000000:01000000:0.0:1789760768.004960:0:5445:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:0.0:1789760768.005010:0:5445:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760768.005022:0:5445:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760768.005032:0:5445:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760768.005057:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075ce00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048658816:192.168.203.113@tcp:101: 00000100:00100000:0.0:1789760768.005155:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e081d840 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.019694:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075ce00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048658816:192.168.203.113@tcp:101: 00010000:00010000:2.0:1789760768.019724:0:5445:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5291f expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760768.019757:0:5445:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c5291f expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760768.019836:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048658944:192.168.203.113@tcp:501: 00000100:00100000:2.0:1789760768.019940:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c7864780 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.029162:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048658944:192.168.203.113@tcp:501: 00000100:00100000:2.0:1789760768.029244:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659072:192.168.203.113@tcp:503: 00000100:00100000:2.0:1789760768.029333:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c7864780 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.040289:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659072:192.168.203.113@tcp:503: 00000100:00100000:2.0:1789760768.041893:0:5917:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5917:1876700048659200:192.168.203.113@tcp:502: 00000100:00100000:2.0:1789760768.042061:0:5917:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c78640c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.053739:0:5917:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5917:1876700048659200:192.168.203.113@tcp:502: 00000020:01000000:2.0:1789760768.053793:0:5917:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760768.053815:0:5917:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760768.053819:0:5917:0:(obd_config.c:1563:class_process_config()) marker 25 (0x1) lustre sys.jobid_var 00000020:00000080:2.0:1789760768.053825:0:5917:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:2.0:1789760768.053863:0:5917:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5917:1876700048659328:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760768.054066:0:5917:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c78640c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.056515:0:5917:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5917:1876700048659328:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:2.0:1789760768.056558:0:5917:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760768.056567:0:5917:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760768.056569:0:5917:0:(obd_config.c:1563:class_process_config()) marker 25 (0x2) lustre sys.jobid_var 00000040:00080000:2.0:1789760768.056573:0:5917:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:0.0:1789760768.059650:0:5445:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 39-41 (rc=0) 10000000:01000000:0.0:1789760768.059681:0:5445:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:0.0:1789760768.059694:0:5445:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5291f expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760768.059705:0:5445:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c0030c00/0x3d1f44aaa2c56217 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5291f expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1789760768.059716:0:5445:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:0.0:1789760768.059719:0:5445:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-sptlrpc-ffff9811c2e4b370 from 1 10000000:01000000:0.0:1789760768.059737:0:5445:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:0.0:1789760768.059774:0:5445:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760768.059783:0:5445:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760768.059794:0:5445:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1789760768.059889:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659456:192.168.203.113@tcp:101:ll_cfg_requeue.0 00000100:00100000:0.0:1789760768.060056:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e081d000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.062128:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659456:192.168.203.113@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1789760768.062154:0:5445:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5296c expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760768.062168:0:5445:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c5296c expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760768.062240:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659584:192.168.203.113@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760768.062312:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c78640c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760768.065335:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048659584:192.168.203.113@tcp:501:ll_cfg_requeue.0 10000000:01000000:2.0:1789760768.065391:0:5445:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:2.0:1789760768.065402:0:5445:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5296c expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760768.065412:0:5445:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c0031c00/0x3d1f44aaa2c5621e lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c5296c expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760768.387654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760769.411830:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760769.541441:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048659840:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760769.541618:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048660096:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760769.542068:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048659712:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760769.548678:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048659968:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760769.548794:0:2352:0:(events.c:340:request_in_callback()) peer: 12345-192.168.203.113@tcp (source: 12345-192.168.203.113@tcp) 00000100:00100000:0.0:1789760769.548937:0:5433:0:(service.c:2301:ptlrpc_server_handle_req_in()) unwrap req x1876700066511744 00000100:00100000:0.0:1789760769.548945:0:5433:0:(service.c:2366:ptlrpc_server_handle_req_in()) got req x1876700066511744 00000100:00100000:0.0:1789760769.549023:0:5433:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.203.113@tcp, seq: 2 00000100:00100000:0.0:1789760769.549038:0:5433:0:(service.c:2552:ptlrpc_server_handle_request()) Handling RPC req@ffff9811e075ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+3:8875:x1876700066511744:12345-192.168.203.113@tcp:104: 00010000:00010000:0.0:1789760769.549057:0:5433:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1789760769.549114:0:5433:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:0.0:1789760769.549120:0:5433:0:(service.c:2610:ptlrpc_server_handle_request()) Handled RPC req@ffff9811e075ce00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea+3:8875:x1876700066511744:12345-192.168.203.113@tcp:104: Request processed in 88us (327us total) trans 0 rc 0/0 00000100:00100000:0.0:1789760769.549131:0:5433:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.203.113@tcp, seq: 2 00010000:00010000:3.0:1789760769.549771:0:5443:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1789760769.549785:0:5443:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760769.549798:0:5443:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9811c1519400 already unused, calling callback (ffffffffc13dac20) 10000000:00010000:3.0:1789760769.549803:0:5443:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760769.549815:0:5443:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1789760769.549834:0:5443:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1789760769.549842:0:5443:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:3.0:1789760769.549902:0:5443:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760769.549910:0:5443:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1789760769.549958:0:5443:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760769.549967:0:5443:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.203.113@tcp lock: ffff9811c1519400/0x3d1f44aaa2c561e6 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0x1287b6a7b7c528e0 expref: -99 pid: 5431 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760769.549994:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760769.549996:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760769.549998:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760769.550022:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048660224:192.168.203.113@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1789760769.552000:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048659840:192.168.203.113@tcp:400:kworker.0 00010000:00080000:0.0:1789760769.557115:0:2360:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e075ca80 x1876700048660096/t0(0) o400->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760785 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760769.557141:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048660096:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760769.557296:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048660224:192.168.203.113@tcp:103:ldlm_bl.0 00000100:00100000:1.0:1789760769.562073:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048659712:192.168.203.113@tcp:400:kworker.0 00010000:00080000:1.0:1789760769.562105:0:2359:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e075f800 x1876700048659968/t0(0) o400->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760785 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760769.562121:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048659968:192.168.203.113@tcp:400:kworker.0 10000000:01000000:2.0:1789760769.562420:0:5445:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:2.0:1789760769.562425:0:5445:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9811e1a82800 from 4 10000000:01000000:2.0:1789760769.562430:0:5445:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1789760769.562555:0:5445:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760769.562567:0:5445:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760769.562581:0:5445:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760769.562636:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660352:192.168.203.113@tcp:101:ll_cfg_requeue.0 00000100:00100000:2.0:1789760769.562790:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c7864780 going to sleep for 11 seconds 00000100:00100000:2.0:1789760769.565807:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660352:192.168.203.113@tcp:101:ll_cfg_requeue.0 00010000:00010000:2.0:1789760769.565850:0:5445:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c52988 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760769.565865:0:5445:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c52988 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760769.565965:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660480:192.168.203.113@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760769.566051:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c7864780 going to sleep for 11 seconds 00000100:00100000:2.0:1789760769.575046:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660480:192.168.203.113@tcp:501:ll_cfg_requeue.0 00000100:00100000:2.0:1789760769.575106:0:5445:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660608:192.168.203.113@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1789760769.575189:0:5445:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c7864780 going to sleep for 11 seconds 00000100:00100000:2.0:1789760769.584518:0:5445:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5445:1876700048660608:192.168.203.113@tcp:503:ll_cfg_requeue.0 00000100:00100000:2.0:1789760769.584749:0:5936:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5936:1876700048660736:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760769.585150:0:5936:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c78640c0 going to sleep for 11 seconds 00000100:00100000:1.0:1789760769.592507:0:5936:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5936:1876700048660736:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:1.0:1789760769.592554:0:5936:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:1.0:1789760769.592564:0:5936:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760769.592567:0:5936:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:1.0:1789760769.592575:0:5936:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9811e1a82800 00000020:00000080:1.0:1789760769.592590:0:5936:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:1.0:1789760769.592673:0:5936:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5936:1876700048660864:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:1.0:1789760769.592750:0:5936:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811d82ca240 going to sleep for 11 seconds 00000100:00100000:2.0:1789760769.601272:0:5936:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:5936:1876700048660864:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:2.0:1789760769.601324:0:5936:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760769.601333:0:5936:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760769.601335:0:5936:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:2.0:1789760769.601339:0:5936:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:2.0:1789760769.602062:0:5445:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:2.0:1789760769.602082:0:5445:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:2.0:1789760769.602093:0:5445:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c52988 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760769.602105:0:5445:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c1391400/0x3d1f44aaa2c56225 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c52988 expref: -99 pid: 5445 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760770.436551:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760771.462809:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760772.484627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760773.511087:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760774.533381:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760774.660973:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2362:1876700048660992:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760774.661289:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760774.661297:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661120:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760774.669312:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661248:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760774.670078:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2362:1876700048660992:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760774.670102:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760774.670109:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661376:192.168.203.113@tcp:400:kworker.0 00010000:00080000:2.0:1789760774.680133:0:2362:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a23800 x1876700048661376/t0(0) o400->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760790 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760774.680158:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661376:192.168.203.113@tcp:400:kworker.0 00010000:00080000:2.0:1789760774.680172:0:2362:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a20000 x1876700048661248/t0(0) o400->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760790 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760774.680182:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661248:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760774.680195:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048661120:192.168.203.113@tcp:400:kworker.0 00000001:02000400:3.0:1789760774.745021:0:6278:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg313-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:3.0:1789760775.555917:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760776.579693:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760777.603581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760778.296162:0:6560:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg313-client.virtnet: executing yml_node 00010000:00010000:3.0:1789760778.627824:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760779.652108:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760779.782765:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048661504:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760779.785420:0:2359:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760779.785451:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048661632:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760779.788626:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048661888:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760779.788745:0:2360:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760779.788755:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048661760:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760779.790458:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048661632:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760779.791545:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048661504:192.168.203.113@tcp:400:kworker.0 00010000:00080000:1.0:1789760779.792551:0:2360:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c5c00 x1876700048661760/t0(0) o400->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760795 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760779.792569:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048661760:192.168.203.113@tcp:400:kworker.0 00010000:00080000:1.0:1789760779.792584:0:2360:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c6d80 x1876700048661888/t0(0) o400->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760795 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760779.792592:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048661888:192.168.203.113@tcp:400:kworker.0 00010000:00010000:0.0:1789760780.675617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1789760781.656093:0:6997:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.57.103 00010000:00010000:2.0:1789760781.702318:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760782.724599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1789760783.502002:0:7152:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.57.103 00010000:00010000:2.0:1789760783.750031:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760784.772274:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760784.901984:0:38:0:(import.c:65:import_set_state_nolock()) ffff9811e071f800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1789760784.901991:0:38:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0000-osc-ffff9811e1a82800: disconnect after 23s idle 00000100:00100000:2.0:1789760784.902083:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2362:1876700048662016:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760784.902173:0:2362:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760784.902178:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048662144:192.168.203.113@tcp:400:kworker.0 00000100:00080000:3.0:1789760784.913751:0:38:0:(import.c:65:import_set_state_nolock()) ffff9811e0719800 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:02000000:3.0:1789760784.913758:0:38:0:(import.c:2101:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff9811e1a82800: disconnect after 23s idle 00000100:00100000:0.0:1789760784.914351:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048662272:192.168.203.113@tcp:9:kworker.0 00000100:00100000:0.0:1789760784.916721:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048662400:192.168.203.113@tcp:9:kworker.0 00000100:00100000:3.0:1789760784.921610:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048662144:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760784.921867:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2362:1876700048662016:192.168.203.113@tcp:400:kworker.0 00010000:00080000:3.0:1789760784.922502:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a22a00 x1876700048662272/t0(0) o9->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760784.922539:0:2358:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff9811e1a22a00 x1876700048662272/t0(0) o9->lustre-OST0000-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760794 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760784.922548:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071f800 lustre-OST0000_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1789760784.922552:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048662272:192.168.203.113@tcp:9:kworker.0 00010000:00080000:3.0:1789760784.922565:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811e1a23100 x1876700048662400/t0(0) o9->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760794 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760784.922575:0:2358:0:(import.c:2011:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff9811e1a23100 x1876700048662400/t0(0) o9->lustre-OST0001-osc-ffff9811e1a82800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760794 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760784.922581:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0719800 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:3.0:1789760784.922585:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048662400:192.168.203.113@tcp:9:kworker.0 00000001:02000400:1.0:1789760785.220008:0:7306:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.57.103 00010000:00010000:2.0:1789760785.797588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760786.571799:0:7451:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: sanityn ============----- Fri Sep 18 15:46:25 EDT 2026 00010000:00010000:0.0:1789760786.820809:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760787.845772:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760788.869889:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760789.891626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760790.021449:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048662656:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760790.021581:0:2361:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1789760790.021589:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048662528:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760790.030148:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048662656:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760790.032531:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048662528:192.168.203.113@tcp:400:kworker.0 00010000:00010000:0.0:1789760790.915838:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760791.940735:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760792.965617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760793.988369:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760795.011677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760795.143680:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048662784:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760795.143792:0:2359:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760795.143799:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048662912:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760795.150424:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048662784:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760795.150538:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048662912:192.168.203.113@tcp:400:kworker.0 00010000:00010000:1.0:1789760796.035768:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760797.074233:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760798.083819:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760799.108935:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1789760799.665680:0:7968:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 40a 00010000:00010000:1.0:1789760800.132599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760800.259815:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048663168:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.259931:0:2361:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760800.259939:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048663040:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.264841:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2361:1876700048663168:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760800.266368:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048663040:192.168.203.113@tcp:400:kworker.0 00000001:02000400:3.0:1789760800.393904:0:8105:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: skipping tests SLOW=no: 33a 00000001:02000400:0.0:1789760801.023031:0:8249:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanityn: start setup 15:46:40 (1789760800) === 00000080:00200008:2.0:1789760801.073875:0:7468:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1789760801.073886:0:7468:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:2.0:1789760801.073907:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760801.073926:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1789760801.073940:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:2.0:1789760801.073948:0:7468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760801.073955:0:7468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:2.0:1789760801.073964:0:7468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:2.0:1789760801.073972:0:7468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e41c4fcb8 00010000:00010000:2.0:1789760801.073977:0:7468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:2.0:1789760801.073986:0:7468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200008:1.0:1789760801.086045:0:7468:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760801.086057:0:7468:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:1.0:1789760801.086076:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760801.086093:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1789760801.086107:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:1.0:1789760801.086116:0:7468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760801.086123:0:7468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:1.0:1789760801.086138:0:7468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:1.0:1789760801.086146:0:7468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e41c4fcb8 00010000:00010000:1.0:1789760801.086150:0:7468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:1.0:1789760801.086223:0:7468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200008:2.0:1789760801.102070:0:7468:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1789760801.102081:0:7468:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:2.0:1789760801.102100:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789760801.102119:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1789760801.102145:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00000080:00200000:2.0:1789760801.102154:0:7468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789760801.102161:0:7468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:2.0:1789760801.102164:0:7468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:2.0:1789760801.102170:0:7468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e41c4fcb8 00010000:00010000:2.0:1789760801.102174:0:7468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 6 lru_type: 1 00010000:00010000:2.0:1789760801.102183:0:7468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200008:1.0:1789760801.113825:0:7468:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1789760801.113837:0:7468:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:1.0:1789760801.113938:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1789760801.113953:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1789760801.113964:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00000080:00200000:1.0:1789760801.113975:0:7468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1789760801.113981:0:7468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:1.0:1789760801.113983:0:7468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:1.0:1789760801.113989:0:7468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e41c4fcb8 00010000:00010000:1.0:1789760801.113991:0:7468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 7 lru_type: 1 00010000:00010000:1.0:1789760801.113999:0:7468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:1.0:1789760801.155618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:01000004:1.0:1789760801.232288:0:8297:0:(obd_mount.c:923:lmd_print()) mount data: 00000020:01000004:1.0:1789760801.232292:0:8297:0:(obd_mount.c:925:lmd_print()) profile: lustre-client 00000020:01000004:1.0:1789760801.232293:0:8297:0:(obd_mount.c:926:lmd_print()) device: 192.168.203.113@tcp:/lustre 00000020:01000004:1.0:1789760801.232295:0:8297:0:(obd_mount.c:929:lmd_print()) options: user_xattr,flock 00000080:01200004:1.0:1789760801.232423:0:8297:0:(super25.c:131:lustre_fill_super()) VFS Op: sb ffff9811e0606800 00000080:01000004:1.0:1789760801.232432:0:8297:0:(super25.c:162:lustre_fill_super()) Mounting client lustre-client 00000020:01000004:1.0:1789760801.232451:0:8297:0:(obd_mount.c:403:lustre_start_mgc()) MGC192.168.203.113@tcp:Set MGC reconnect 1 10000000:01000000:1.0:1789760801.232454:0:8297:0:(mgc_request.c:1071:mgc_set_info_async()) InitRecov MGC192.168.203.113@tcp 1/d0:i0:r0:or0:FULL 00000080:00200000:1.0:1789760801.232485:0:8297:0:(llite_lib.c:1506:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff9811e0606800 (sb ffff9811e0606800) 00000080:01000000:1.0:1789760801.232561:0:8297:0:(llite_lib.c:1232:ll_options()) Parsing opts user_xattr,flock 00000080:00000004:1.0:1789760801.232563:0:8297:0:(llite_lib.c:1242:ll_options()) next opt=user_xattr 00000080:00000004:1.0:1789760801.232566:0:8297:0:(llite_lib.c:1242:ll_options()) next opt=flock 00000080:01000000:1.0:1789760801.232571:0:8297:0:(llite_lib.c:1540:ll_fill_super()) llite sb uuid: cfa8bac1-156b-47a9-afab-fe5753451799 10000000:01000000:1.0:1789760801.233070:0:8297:0:(mgc_request.c:1944:mgc_process_config()) parse_log lustre-client from 0 10000000:01000000:1.0:1789760801.233074:0:8297:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff9811e0606800 10000000:01000000:1.0:1789760801.233081:0:8297:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff9811e0606800 10000000:01000000:1.0:1789760801.233086:0:8297:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 10000000:01000000:1.0:1789760801.233091:0:8297:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff9811e0606800 10000000:01000000:1.0:1789760801.233095:0:8297:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 10000000:01000000:1.0:1789760801.233103:0:8297:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff9811e0606800 10000000:01000000:1.0:1789760801.233108:0:8297:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 10000000:01000000:1.0:1789760801.233112:0:8297:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-client-ffff9811e0606800 from 1 10000000:01000000:1.0:1789760801.233116:0:8297:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:1.0:1789760801.233167:0:8297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760801.233179:0:8297:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1789760801.233192:0:8297:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1789760801.233215:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663296:192.168.203.113@tcp:101:mount.lustre.0 00000100:00100000:1.0:1789760801.233303:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6c5bf00 going to sleep for 11 seconds 00000100:00100000:3.0:1789760801.235256:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663296:192.168.203.113@tcp:101:mount.lustre.0 00010000:00010000:3.0:1789760801.235281:0:8297:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529b9 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760801.235311:0:8297:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c529b9 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1789760801.235385:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663424:192.168.203.113@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760801.235444:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c2a7fd80 going to sleep for 11 seconds 00000100:00100000:3.0:1789760801.237106:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663424:192.168.203.113@tcp:501:mount.lustre.0 00000100:00100000:3.0:1789760801.237170:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663552:192.168.203.113@tcp:503:mount.lustre.0 00000100:00100000:3.0:1789760801.237241:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c2a7fd80 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.238323:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663552:192.168.203.113@tcp:503:mount.lustre.0 00000100:00100000:0.0:1789760801.239369:0:8301:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8301:1876700048663680:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:0.0:1789760801.239431:0:8301:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e123a600 going to sleep for 11 seconds 00000100:00100000:0.0:1789760801.240810:0:8301:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8301:1876700048663680:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:0.0:1789760801.240846:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.240856:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.240859:0:8301:0:(obd_config.c:1563:class_process_config()) marker 4 (0x1) lustre-clilov lov setup 00000020:01000000:0.0:1789760801.240866:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilov-ffff9811e0606800 00000020:00000080:0.0:1789760801.240869:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760801.240882:0:8301:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilov-ffff9811e0606800 (ffff9811c2cfcd28) 00000020:00000080:0.0:1789760801.240902:0:8301:0:(obd_config.c:720:class_attach_name()) OBD: dev 6 attached type lov with refcount 2 00000020:01000000:0.0:1789760801.240910:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilov-ffff9811e0606800 00000020:00000080:0.0:1789760801.240913:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:0.0:1789760801.241225:0:8301:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilov-ffff9811e0606800 (uuid cfa8bac1-156b-47a9-afab-fe5753451799) 00000020:01000000:0.0:1789760801.241231:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.241235:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.241237:0:8301:0:(obd_config.c:1563:class_process_config()) marker 4 (0x2) lustre-clilov lov setup 00000020:01000000:0.0:1789760801.241240:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.241243:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.241244:0:8301:0:(obd_config.c:1563:class_process_config()) marker 5 (0x1) lustre-clilmv lmv setup 00000020:01000000:0.0:1789760801.241248:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-clilmv-ffff9811e0606800 00000020:00000080:0.0:1789760801.241251:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760801.241258:0:8301:0:(genops.c:413:class_newdev()) Allocate new device lustre-clilmv-ffff9811e0606800 (ffff9811c2cf99b8) 00000020:00000080:0.0:1789760801.241269:0:8301:0:(obd_config.c:720:class_attach_name()) OBD: dev 7 attached type lmv with refcount 2 00000020:01000000:0.0:1789760801.241274:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-clilmv-ffff9811e0606800 00000020:00000080:0.0:1789760801.241277:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00000020:00000080:0.0:1789760801.242311:0:8301:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-clilmv-ffff9811e0606800 (uuid cfa8bac1-156b-47a9-afab-fe5753451799) 00000020:01000000:0.0:1789760801.242318:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.242322:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.242323:0:8301:0:(obd_config.c:1563:class_process_config()) marker 5 (0x2) lustre-clilmv lmv setup 00000020:01000000:0.0:1789760801.242327:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.242330:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.242332:0:8301:0:(obd_config.c:1563:class_process_config()) marker 6 (0x1) lustre-client mount opts 00000020:00000080:0.0:1789760801.242336:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf007 00000020:00000080:0.0:1789760801.242338:0:8301:0:(obd_config.c:1516:class_process_config()) mountopt: profile lustre-client osc lustre-clilov mdc lustre-clilmv 00000020:01000000:0.0:1789760801.242344:0:8301:0:(obd_config.c:1149:class_add_profile()) Add profile lustre-client 00000020:01000000:0.0:1789760801.242350:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.242354:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.242356:0:8301:0:(obd_config.c:1563:class_process_config()) marker 6 (0x2) lustre-client mount opts 00000020:01000000:0.0:1789760801.242358:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.242362:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.242363:0:8301:0:(obd_config.c:1563:class_process_config()) marker 7 (0x1) lustre-MDT0000 add mdc 00000020:00000080:0.0:1789760801.242367:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760801.242372:0:8301:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760801.242394:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-MDT0000-mdc-ffff9811e0606800 00000020:00000080:0.0:1789760801.242397:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760801.242405:0:8301:0:(genops.c:413:class_newdev()) Allocate new device lustre-MDT0000-mdc-ffff9811e0606800 (ffff9811c2cfb370) 00000020:00000080:0.0:1789760801.242415:0:8301:0:(obd_config.c:720:class_attach_name()) OBD: dev 8 attached type mdc with refcount 2 00000020:01000000:0.0:1789760801.242420:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-MDT0000-mdc-ffff9811e0606800 00000020:00000080:0.0:1789760801.242422:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760801.242578:0:8301:0:(ldlm_lib.c:102:import_set_conn()) imp ffff9811e0717000@lustre-MDT0000-mdc-ffff9811e0606800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760801.244401:0:8301:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-MDT0000-mdc-ffff9811e0606800-4 is initialized 00000040:01000000:0.0:1789760801.244742:0:8301:0:(llog_obd.c:197:llog_setup()) obd lustre-MDT0000-mdc-ffff9811e0606800 ctxt 13 is initialized 00000020:00000080:0.0:1789760801.244780:0:8301:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-MDT0000-mdc-ffff9811e0606800 (uuid cfa8bac1-156b-47a9-afab-fe5753451799) 00000020:01000000:0.0:1789760801.244792:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf014, instance name: lustre-clilmv-ffff9811e0606800 00000020:00000080:0.0:1789760801.244796:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf014 00800000:01000000:0.0:1789760801.244816:0:8301:0:(lmv_obd.c:442:lmv_add_target()) Target uuid: lustre-MDT0000_UUID. index 0 00000020:01000000:0.0:1789760801.244858:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.244863:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.244865:0:8301:0:(obd_config.c:1563:class_process_config()) marker 7 (0x2) lustre-MDT0000 add mdc 00000020:01000000:0.0:1789760801.244868:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1789760801.244870:0:8301:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #8 00000020:01000000:0.0:1789760801.244872:0:8301:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760801.244874:0:8301:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760801.244875:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1789760801.244879:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.244880:0:8301:0:(obd_config.c:1563:class_process_config()) marker 8 (0x6) lustre sys.timeout 00000020:01000000:0.0:1789760801.244889:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1789760801.244893:0:8301:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0000 (0) in 0 of 192.168.203.113@tcp:/lustre 00000020:00000080:0.0:1789760801.244896:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.244897:0:8301:0:(obd_config.c:1563:class_process_config()) marker 13 (0x1) lustre-OST0000 add osc 00000020:00000080:0.0:1789760801.244901:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760801.244904:0:8301:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760801.244924:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0000-osc-ffff9811e0606800 00000020:00000080:0.0:1789760801.244928:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760801.244945:0:8301:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0000-osc-ffff9811e0606800 (ffff9811c2ec0000) 00000020:00000080:0.0:1789760801.244958:0:8301:0:(obd_config.c:720:class_attach_name()) OBD: dev 9 attached type osc with refcount 2 00000020:01000000:0.0:1789760801.244964:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0000-osc-ffff9811e0606800 00000020:00000080:0.0:1789760801.244968:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760801.245051:0:8301:0:(ldlm_lib.c:102:import_set_conn()) imp ffff9811c2ed5800@lustre-OST0000-osc-ffff9811e0606800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760801.245269:0:8301:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0000-osc-ffff9811e0606800-5 is initialized 00000020:00000080:0.0:1789760801.245522:0:8301:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0000-osc-ffff9811e0606800 (uuid cfa8bac1-156b-47a9-afab-fe5753451799) 00000020:01000000:0.0:1789760801.245530:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9811e0606800 00000020:00000080:0.0:1789760801.245533:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1789760801.245544:0:8301:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0000_UUID idx:0 gen:1 active:1 00020000:01000000:0.0:1789760801.245550:0:8301:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9811e0606800: Adding target lustre-OST0000_UUID 00020000:01000000:0.0:1789760801.245554:0:8301:0:(lov_obd.c:532:lov_add_target()) idx=0 ltd_gen=1 ld_tgt_count=1 00000020:01000000:0.0:1789760801.245569:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.245572:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.245574:0:8301:0:(obd_config.c:1563:class_process_config()) marker 13 (0x2) lustre-OST0000 add osc 00000020:01000000:0.0:1789760801.245578:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x5 00000020:01000000:0.0:1789760801.245580:0:8301:0:(obd_config.c:1902:class_config_llog_handler()) SKIP #15 00000020:01000000:0.0:1789760801.245582:0:8301:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760801.245583:0:8301:0:(obd_config.c:1928:class_config_llog_handler()) skipping 0x6 00000020:01000000:0.0:1789760801.245584:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x6 mark_flg=0x6 00000020:00000080:0.0:1789760801.245588:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.245589:0:8301:0:(obd_config.c:1563:class_process_config()) marker 15 (0x6) lustre sys.timeout 00000020:01000000:0.0:1789760801.245592:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:01000004:0.0:1789760801.245595:0:8301:0:(obd_mount.c:961:lustre_check_exclusion()) Check exclusion lustre-OST0001 (1) in 0 of 192.168.203.113@tcp:/lustre 00000020:00000080:0.0:1789760801.245604:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.245606:0:8301:0:(obd_config.c:1563:class_process_config()) marker 19 (0x1) lustre-OST0001 add osc 00000020:00000080:0.0:1789760801.245612:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1789760801.245615:0:8301:0:(obd_config.c:1487:class_process_config()) adding mapping from uuid 192.168.203.113@tcp to nid 0x20000c0a8cb71 (192.168.203.113@tcp) 00000020:01000000:0.0:1789760801.245629:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf001, instance name: lustre-OST0001-osc-ffff9811e0606800 00000020:00000080:0.0:1789760801.245632:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf001 00000020:00000080:0.0:1789760801.245638:0:8301:0:(genops.c:413:class_newdev()) Allocate new device lustre-OST0001-osc-ffff9811e0606800 (ffff9811c2ec4d28) 00000020:00000080:0.0:1789760801.245648:0:8301:0:(obd_config.c:720:class_attach_name()) OBD: dev 10 attached type osc with refcount 2 00000020:01000000:0.0:1789760801.245653:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf003, instance name: lustre-OST0001-osc-ffff9811e0606800 00000020:00000080:0.0:1789760801.245656:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf003 00010000:00080000:0.0:1789760801.245754:0:8301:0:(ldlm_lib.c:102:import_set_conn()) imp ffff9811c2ed2000@lustre-OST0001-osc-ffff9811e0606800: add connection 192.168.203.113@tcp at head 00010000:00010000:0.0:1789760801.246073:0:8301:0:(ldlm_pool.c:923:ldlm_pool_init()) Lock pool ldlm-pool-lustre-OST0001-osc-ffff9811e0606800-6 is initialized 00000020:00000080:0.0:1789760801.246455:0:8301:0:(obd_config.c:826:class_setup()) finished setup of obd lustre-OST0001-osc-ffff9811e0606800 (uuid cfa8bac1-156b-47a9-afab-fe5753451799) 00000020:01000000:0.0:1789760801.246512:0:8301:0:(obd_config.c:2004:class_config_llog_handler()) cmd cf00d, instance name: lustre-clilov-ffff9811e0606800 00000020:00000080:0.0:1789760801.246516:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00d 00020000:01000000:0.0:1789760801.246528:0:8301:0:(lov_obd.c:481:lov_add_target()) uuid:lustre-OST0001_UUID idx:1 gen:1 active:1 00020000:01000000:0.0:1789760801.246535:0:8301:0:(lov_obd.c:510:lov_add_target()) lustre-clilov-ffff9811e0606800: Adding target lustre-OST0001_UUID 00020000:01000000:0.0:1789760801.246538:0:8301:0:(lov_obd.c:532:lov_add_target()) idx=1 ltd_gen=1 ld_tgt_count=2 00000020:01000000:0.0:1789760801.246552:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.246556:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.246558:0:8301:0:(obd_config.c:1563:class_process_config()) marker 19 (0x2) lustre-OST0001 add osc 00000020:01000000:0.0:1789760801.246562:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.246566:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.246567:0:8301:0:(obd_config.c:1563:class_process_config()) marker 21 (0x1) lustre sys.timeout 00000020:00000080:0.0:1789760801.246572:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf009 00000020:00000080:0.0:1789760801.246574:0:8301:0:(obd_config.c:1537:class_process_config()) changing lustre timeout from 20 to 20 00000020:01000000:0.0:1789760801.246577:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:0.0:1789760801.246581:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.246583:0:8301:0:(obd_config.c:1563:class_process_config()) marker 21 (0x2) lustre sys.timeout 00000020:01000000:0.0:1789760801.246586:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:0.0:1789760801.246590:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:0.0:1789760801.246591:0:8301:0:(obd_config.c:1563:class_process_config()) marker 25 (0x1) lustre sys.jobid_var 00000020:00000080:0.0:1789760801.246602:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf00f 00000100:00100000:0.0:1789760801.246646:0:8301:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8301:1876700048663808:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:0.0:1789760801.246746:0:8301:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e123a600 going to sleep for 11 seconds 00000100:00100000:1.0:1789760801.250733:0:8301:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8301:1876700048663808:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:1.0:1789760801.250779:0:8301:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:1.0:1789760801.250788:0:8301:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:1.0:1789760801.250790:0:8301:0:(obd_config.c:1563:class_process_config()) marker 25 (0x2) lustre sys.jobid_var 00000040:00080000:1.0:1789760801.250795:0:8301:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 42 count 42 00000020:01000000:2.0:1789760801.250901:0:8297:0:(obd_config.c:2145:class_config_parse_llog()) Processed log lustre-client gen 1-41 (rc=0) 10000000:01000000:2.0:1789760801.250927:0:8297:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:2.0:1789760801.250947:0:8297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529b9 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.250959:0:8297:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9812021fc800/0x3d1f44aaa2c5622c lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529b9 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760801.250968:0:8297:0:(mgc_request.c:1802:mgc_process_log()) Process log lustre-cliir-ffff9811e0606800 from 1 10000000:01000000:2.0:1789760801.250971:0:8297:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-cliir (res 0x65727473756c) 00010000:00010000:2.0:1789760801.251000:0:8297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.251009:0:8297:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.251019:0:8297:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760801.251038:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663936:192.168.203.113@tcp:101:mount.lustre.0 00000100:00100000:2.0:1789760801.251106:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.256027:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048663936:192.168.203.113@tcp:101:mount.lustre.0 00010000:00010000:2.0:1789760801.256063:0:8297:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.256079:0:8297:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c529c0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760801.256661:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664064:192.168.203.113@tcp:256:mount.lustre.0 00000100:00100000:2.0:1789760801.256758:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.259030:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664064:192.168.203.113@tcp:256:mount.lustre.0 10000000:01000000:2.0:1789760801.259803:0:8297:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'lustre-cliir' succeeded (0). 00010000:00010000:2.0:1789760801.259827:0:8297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.259840:0:8297:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c1390e00/0x3d1f44aaa2c56256 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1789760801.259849:0:8297:0:(mgc_request.c:1802:mgc_process_log()) Process log params-ffff9811e0606800 from 1 10000000:01000000:2.0:1789760801.259852:0:8297:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:2.0:1789760801.259891:0:8297:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.259899:0:8297:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.259911:0:8297:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760801.259935:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664192:192.168.203.113@tcp:101:mount.lustre.0 00000100:00100000:2.0:1789760801.260010:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.261439:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664192:192.168.203.113@tcp:101:mount.lustre.0 00010000:00010000:2.0:1789760801.261498:0:8297:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c7 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789760801.261512:0:8297:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x1287b6a7b7c529c7 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789760801.261623:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664320:192.168.203.113@tcp:501:mount.lustre.0 00000100:00100000:2.0:1789760801.261678:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.262986:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664320:192.168.203.113@tcp:501:mount.lustre.0 00000100:00100000:2.0:1789760801.263031:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664448:192.168.203.113@tcp:503:mount.lustre.0 00000100:00100000:2.0:1789760801.263105:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48000 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.264663:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8297:1876700048664448:192.168.203.113@tcp:503:mount.lustre.0 00000100:00100000:2.0:1789760801.265151:0:8303:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8303:1876700048664576:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760801.265225:0:8303:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48e40 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.266444:0:8303:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8303:1876700048664576:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:2.0:1789760801.266513:0:8303:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760801.266534:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760801.266537:0:8303:0:(obd_config.c:1563:class_process_config()) marker 2 (0x1) general debug_raw_pointers 00000020:01000000:2.0:1789760801.266543:0:8303:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9811e0606800 00000020:00000080:2.0:1789760801.266546:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000020:01000000:2.0:1789760801.266556:0:8303:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760801.266557:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760801.266559:0:8303:0:(obd_config.c:1563:class_process_config()) marker 2 (0x2) general debug_raw_pointers 00000020:01000000:2.0:1789760801.266561:0:8303:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1789760801.266564:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760801.266565:0:8303:0:(obd_config.c:1563:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:2.0:1789760801.266568:0:8303:0:(obd_config.c:2004:class_config_llog_handler()) cmd ce032, instance name: general-ffff9811e0606800 00000020:00000080:2.0:1789760801.266570:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: ce032 00000100:00100000:2.0:1789760801.266660:0:8303:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8303:1876700048664704:192.168.203.113@tcp:502:llog_process_th.0 00000100:00100000:2.0:1789760801.266722:0:8303:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48e40 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.267972:0:8303:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job llog_process_th:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:8303:1876700048664704:192.168.203.113@tcp:502:llog_process_th.0 00000020:01000000:2.0:1789760801.268005:0:8303:0:(obd_config.c:1893:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1789760801.268012:0:8303:0:(obd_config.c:1472:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1789760801.268014:0:8303:0:(obd_config.c:1563:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:2.0:1789760801.268019:0:8303:0:(llog.c:862:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:3.0:1789760801.268166:0:8297:0:(obd_config.c:2145:class_config_parse_llog()) Processed log params gen 1-6 (rc=0) 10000000:01000000:3.0:1789760801.268199:0:8297:0:(mgc_request.c:1879:mgc_process_log()) MGC192.168.203.113@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:3.0:1789760801.268222:0:8297:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c7 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1789760801.268236:0:8297:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.203.113@tcp lock: ffff9811c1391600/0x3d1f44aaa2c5625d lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0x1287b6a7b7c529c7 expref: -99 pid: 8297 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:01000000:3.0:1789760801.268262:0:8297:0:(llite_lib.c:1614:ll_fill_super()) Found profile lustre-client: mdc=lustre-clilmv osc=lustre-clilov 00000020:00000080:3.0:1789760801.268292:0:8297:0:(genops.c:1385:class_connect()) connect: client cfa8bac1-156b-47a9-afab-fe5753451799, cookie 0x3d1f44aaa2c56264 00800000:01000000:3.0:1789760801.268319:0:8297:0:(lmv_obd.c:515:lmv_check_connect()) Time to connect cfa8bac1-156b-47a9-afab-fe5753451799 to lustre-clilmv-ffff9811e0606800 00800000:01000000:3.0:1789760801.268328:0:8297:0:(lmv_obd.c:338:lmv_connect_mdc()) connect to lustre-MDT0000-mdc-ffff9811e0606800(cfa8bac1-156b-47a9-afab-fe5753451799) - lustre-MDT0000_UUID, cfa8bac1-156b-47a9-afab-fe5753451799 00000020:00000080:3.0:1789760801.268337:0:8297:0:(genops.c:1385:class_connect()) connect: client cfa8bac1-156b-47a9-afab-fe5753451799, cookie 0x3d1f44aaa2c5626b 00000100:00080000:3.0:1789760801.268344:0:8297:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:3.0:1789760801.268351:0:8297:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760801.268371:0:8297:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:3.0:1789760801.268408:0:8297:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048664832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760801.268440:0:8297:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID 00000100:00100000:2.0:1789760801.268510:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048664832:192.168.203.113@tcp:38:mount.lustre.0 00000100:00100000:0.0:1789760801.268558:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048664960:192.168.203.113@tcp:400:kworker.0 00800000:01000000:3.0:1789760801.268593:0:8297:0:(lmv_obd.c:392:lmv_connect_mdc()) Connected to lustre-MDT0000-mdc-ffff9811e0606800(cfa8bac1-156b-47a9-afab-fe5753451799) successfully (5) 00800000:00000004:3.0:1789760801.268598:0:8297:0:(obd_class.h:1056:obd_statfs_async()) lustre-MDT0000-mdc-ffff9811e0606800: age -745, max_age 0 00000100:00100000:2.0:1789760801.268646:0:2361:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:2.0:1789760801.268663:0:2361:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9811e1a22300 x1876700048665088/t0(0) o41->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000080:00000004:3.0:1789760801.268725:0:8297:0:(obd_class.h:1090:obd_statfs()) lustre-clilmv-ffff9811e0606800: age -745, max_age 254 00800000:00000004:3.0:1789760801.268733:0:8297:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff9811e0606800: age -745, max_age 254 00000100:00080000:3.0:1789760801.268762:0:8297:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff9811e1a20700 x1876700048665216/t0(0) o41->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760801.268770:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c2a7f300 going to sleep for 0 seconds 00000100:00100000:0.0:1789760801.269826:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048664960:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760801.270337:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff9811e0606800: committing for last_committed 0 gen 1 00000100:00080000:2.0:1789760801.270348:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-MDT0000-mdc-ffff9811e0606800: connect to target with instance 1 00000100:00080000:2.0:1789760801.270355:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-MDT0000-mdc-ffff9811e0606800: Resetting ns_connect_flags to server flags: 0xae7a7e7be344d3b8 00000100:00080000:2.0:1789760801.270365:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-MDT0000_UUID 00000100:00080000:2.0:1789760801.270368:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to FULL 00000100:00080000:3.0:1789760801.270654:0:2358:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 00000000c0f4993f): req@ffff9811e1a22300 x1876700048665088/t0(0) o41->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760801.270683:0:2358:0:(recover.c:191:ptlrpc_wake_delayed()) @@@ waking (set 0000000026f22cfa): req@ffff9811e1a20700 x1876700048665216/t0(0) o41->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760801.270717:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048664832:192.168.203.113@tcp:38:mount.lustre.0 00000100:00100000:0.0:1789760801.270945:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c2a7f300 going to sleep for 11 seconds 00000100:00100000:3.0:1789760801.272287:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:cfa8bac1-156b-47a9-afab-fe5753451799:0:1876700048665088:192.168.203.113@tcp:41:mount.lustre.0 00000100:00100000:2.0:1789760801.272346:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job mount.lustre:cfa8bac1-156b-47a9-afab-fe5753451799:8297:1876700048665216:192.168.203.113@tcp:41:mount.lustre.0 00800000:00000004:2.0:1789760801.272371:0:8297:0:(obd_class.h:1116:obd_statfs()) lustre-MDT0000-mdc-ffff9811e0606800: update ffff9811c2cfb7c0 cache blocks 317407/353529 objects 1023725/1024000 00000080:00000004:2.0:1789760801.272378:0:8297:0:(obd_class.h:1116:obd_statfs()) lustre-clilmv-ffff9811e0606800: update ffff9811c2cf9e08 cache blocks 317407/353529 objects 1023725/1024000 00000100:00100000:2.0:1789760801.272412:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:cfa8bac1-156b-47a9-afab-fe5753451799:8297:1876700048665344:192.168.203.113@tcp:40:mount.lustre.0 00000100:00100000:2.0:1789760801.272498:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.274252:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job mount.lustre:cfa8bac1-156b-47a9-afab-fe5753451799:8297:1876700048665344:192.168.203.113@tcp:40:mount.lustre.0 00000080:00000004:2.0:1789760801.274274:0:8297:0:(llite_lib.c:614:client_common_fill_super()) rootfid [0x200000007:0x1:0x0] 00000080:00100000:2.0:1789760801.274300:0:8297:0:(llite_lib.c:775:client_common_fill_super()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 2146304 00020000:01000000:2.0:1789760801.274305:0:8297:0:(lov_obd.c:186:lov_connect()) connect #0 00000020:00000080:2.0:1789760801.274318:0:8297:0:(genops.c:1385:class_connect()) connect: client cfa8bac1-156b-47a9-afab-fe5753451799, cookie 0x3d1f44aaa2c56272 00000020:00000080:2.0:1789760801.274346:0:8297:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x3d1f44aaa2c56279 00000100:00080000:2.0:1789760801.274354:0:8297:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:2.0:1789760801.274363:0:8297:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760801.274374:0:8297:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760801.274401:0:8297:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048665472/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760801.274437:0:8297:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID 00020000:01000000:2.0:1789760801.274492:0:8297:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 0 lustre-OST0000_UUID (lustre-OST0000-osc-ffff9811e0606800) active 00000100:00100000:0.0:1789760801.274506:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048665472:192.168.203.113@tcp:8:mount.lustre.0 00000020:00000080:2.0:1789760801.274532:0:8297:0:(genops.c:1385:class_connect()) connect: client LOV_OSC_UUID, cookie 0x3d1f44aaa2c56280 00000100:00080000:2.0:1789760801.274541:0:8297:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 0 00000100:00080000:2.0:1789760801.274548:0:8297:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760801.274554:0:8297:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 134217728, lost: 0. 00000100:00100000:2.0:1789760801.274574:0:8297:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048665600/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760801.274586:0:8297:0:(pinger.c:373:ptlrpc_pinger_add_import()) adding pingable import cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID 00020000:01000000:2.0:1789760801.274598:0:8297:0:(lov_obd.c:159:lov_connect_osc()) Connected tgt idx 1 lustre-OST0001_UUID (lustre-OST0001-osc-ffff9811e0606800) active 00000100:00100000:0.0:1789760801.274631:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048665600:192.168.203.113@tcp:8:mount.lustre.0 00000100:00100000:2.0:1789760801.274652:0:8297:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job mount.lustre:cfa8bac1-156b-47a9-afab-fe5753451799:8297:1876700048665728:192.168.203.113@tcp:33:mount.lustre.0 00000100:00100000:2.0:1789760801.274776:0:8297:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811c6d48cc0 going to sleep for 11 seconds 00000100:00100000:2.0:1789760801.277941:0:8297:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job mount.lustre:cfa8bac1-156b-47a9-afab-fe5753451799:8297:1876700048665728:192.168.203.113@tcp:33:mount.lustre.0 00010000:00080000:0.0:1789760801.278025:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d863a680 x1876700048665600/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760801.278034:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0001-osc-ffff9811e0606800: committing for last_committed 0 gen 1 00000100:00080000:0.0:1789760801.278041:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0001-osc-ffff9811e0606800: connect to target with instance 1 00000100:00080000:0.0:1789760801.278048:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0001-osc-ffff9811e0606800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1789760801.278056:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0001_UUID 00000100:00080000:0.0:1789760801.278060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to FULL 00020000:01000000:0.0:1789760801.278064:0:2358:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID active 00000080:00000004:0.0:1789760801.278069:0:2358:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0467af2e3640038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1789760801.278074:0:2358:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000080:00200000:2.0:1789760801.278081:0:8297:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d1cb08) 00000100:00100000:0.0:1789760801.278085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048665600:192.168.203.113@tcp:8:mount.lustre.0 00000080:00200000:2.0:1789760801.278090:0:8297:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 4096 00010000:00080000:0.0:1789760801.278115:0:2358:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811d8639880 x1876700048665472/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/416 e 0 to 0 dl 1789760811 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760801.278133:0:2358:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff9811e0606800: committing for last_committed 0 gen 1 00000100:00080000:0.0:1789760801.278138:0:2358:0:(import.c:1132:ptlrpc_connect_interpret()) lustre-OST0000-osc-ffff9811e0606800: connect to target with instance 1 00000100:00080000:0.0:1789760801.278165:0:2358:0:(import.c:956:ptlrpc_connect_set_flags()) lustre-OST0000-osc-ffff9811e0606800: Resetting ns_connect_flags to server flags: 0xa0427af2e3440038 00000100:00080000:0.0:1789760801.278173:0:2358:0:(import.c:1213:ptlrpc_connect_interpret()) connected to replayable target: lustre-OST0000_UUID 00000100:00080000:0.0:1789760801.278176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to FULL 00000080:02000400:2.0:1789760801.278177:0:8297:0:(llite_lib.c:1653:ll_fill_super()) Mounted lustre-client - version 2.17.57_103_g082f77f 00020000:01000000:0.0:1789760801.278182:0:2358:0:(lov_obd.c:398:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID active 00000080:00000004:0.0:1789760801.278187:0:2358:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0427af2e3440038 -> 0xa0427af2e3440038 00000080:00080000:0.0:1789760801.278191:0:2358:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:0.0:1789760801.278202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048665472:192.168.203.113@tcp:8:mount.lustre.0 00000080:00000004:2.0:1789760801.280660:0:8297:0:(super25.c:182:lustre_fill_super()) 192.168.203.113@tcp:/lustre: Mount complete 00010000:00010000:3.0:1789760802.179727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760803.203605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:1.0:1789760803.299120:0:8558:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg313-client.virtnet: executing check_config_client /mnt/lustre 00010000:00010000:3.0:1789760804.227534:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760805.251622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760806.275567:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760806.403653:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048665856:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760806.403653:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048665984:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760806.403744:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048666240:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760806.403757:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048666112:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760806.403793:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048666368:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760806.404891:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048665856:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760806.404915:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048666112:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760806.404915:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048665984:192.168.203.113@tcp:400:kworker.0 00010000:00080000:0.0:1789760806.404929:0:2360:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c4380 x1876700048666368/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760822 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760806.404943:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048666368:192.168.203.113@tcp:400:kworker.0 00010000:00080000:1.0:1789760806.404959:0:2359:0:(ldlm_request.c:1671:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff9811c92c5500 x1876700048666240/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 0 dl 1789760822 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760806.404972:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048666240:192.168.203.113@tcp:400:kworker.0 00010000:00010000:1.0:1789760807.299579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760808.323635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760809.347603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760810.371629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760811.395589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760811.523687:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048666624:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760811.523687:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048666496:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760811.523770:0:2362:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:cfa8bac1-156b-47a9-afab-fe5753451799:2362:1876700048666880:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760811.523771:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:cfa8bac1-156b-47a9-afab-fe5753451799:2361:1876700048666752:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760811.523804:0:2361:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:cfa8bac1-156b-47a9-afab-fe5753451799:2361:1876700048667008:192.168.203.113@tcp:400:kworker.0 00010000:00010000:3.0:1789760812.419631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760813.443618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760814.467613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760815.491590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760816.515663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760816.643643:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048667264:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760816.643644:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048667136:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760816.643732:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048667392:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760816.643747:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048667520:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760816.643780:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e075f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048667648:192.168.203.113@tcp:400:kworker.0 00010000:00010000:3.0:1789760817.539662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1789760818.067452:0:9212:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT= 00010000:00010000:1.0:1789760818.563631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760819.587562:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760820.611611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760821.635628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760821.763742:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048667776:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760821.763746:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048667904:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760821.763876:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048668160:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760821.763889:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668032:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760821.763958:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668288:192.168.203.113@tcp:400:kworker.0 00010000:00010000:3.0:1789760822.659586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760823.683626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760824.707578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760825.731632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760826.755607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760826.883713:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048668416:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760826.883722:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048668544:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760826.883838:0:2359:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048668672:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760826.883839:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668800:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760826.883904:0:2360:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760826.883914:0:2360:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668928:192.168.203.113@tcp:400:kworker.0 00010000:00010000:3.0:1789760827.779613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1789760827.908673:0:2361:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760811/real 1789760811] req@ffff9811d8639500 x1876700048666496/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.920205:0:2361:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811d8639500 x1876700048666496/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760827.920218:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1789760827.920222:0:2361:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.113@tcp: Connection to MGS (at 192.168.203.113@tcp) was lost; in progress operations using this service will fail 00000100:00000400:2.0:1789760827.920516:0:2362:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760811/real 1789760811] req@ffff9811d8638380 x1876700048666624/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760827.920537:0:2362:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811d8638380 x1876700048666624/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760827.920545:0:2362:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1789760827.920549:0:2362:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff9811e1a82800: Connection to lustre-MDT0000 (at 192.168.203.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789760827.925308:0:2361:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.113@tcp for MGC192.168.203.113@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1789760827.925312:0:2361:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1789760827.925319:0:2361:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9811d8639500 x1876700048666496/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.925332:0:2361:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9811e075e300 x1876700048667136/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760832 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.925362:0:2361:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9811c92c5500 x1876700048667776/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760837 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.925370:0:2361:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9811c2e29500 x1876700048668416/t0(0) o400->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 224/224 e 0 to 0 dl 1789760842 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1789760827.925378:0:2361:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1789760827.925380:0:2361:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1789760827.925381:0:2361:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1789760827.925383:0:2361:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:0.0:1789760827.925490:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048668416:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760827.925505:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2359:1876700048667136:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760827.925544:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2360:1876700048667776:192.168.203.113@tcp:400:kworker.0 00000100:00080000:0.0:1789760827.925568:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.925573:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760827.925576:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760827.925582:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 212 00000100:00080000:0.0:1789760827.925595:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760827.925637:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048669056/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760827.926550:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048669056:192.168.203.113@tcp:250:kworker.0 00000100:00000400:3.0:1789760827.926769:0:2361:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760811/real 1789760811] req@ffff9811d8639f80 x1876700048666752/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.926799:0:2361:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811d8639f80 x1876700048666752/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760827.926806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789760827.926809:0:2361:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff9811e0606800: Connection to lustre-MDT0000 (at 192.168.203.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789760827.926815:0:2361:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1789760827.926829:0:2361:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760811/real 1789760811] req@ffff9811d8638a80 x1876700048667008/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760827.926842:0:2361:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811d8638a80 x1876700048667008/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760827.926847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789760827.926850:0:2361:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff9811e0606800: Connection to lustre-OST0001 (at 192.168.203.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789760827.926854:0:2361:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1789760827.926866:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2361:1876700048666496:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760827.926873:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:cfa8bac1-156b-47a9-afab-fe5753451799:2361:1876700048666752:192.168.203.113@tcp:400:kworker.0 00000100:00100000:3.0:1789760827.926877:0:2361:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:cfa8bac1-156b-47a9-afab-fe5753451799:2361:1876700048667008:192.168.203.113@tcp:400:kworker.0 00000100:00080000:0.0:1789760827.933155:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760827.933155:0:2362:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00080000:0.0:1789760827.933163:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760827.933166:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760827.933172:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 212 00000100:00000400:2.0:1789760827.933178:0:2362:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760811/real 1789760811] req@ffff9811d8638e00 x1876700048666880/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760827.933183:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760827.933218:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760827.933219:0:2362:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811d8638e00 x1876700048666880/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760827 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760827.933231:0:2362:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1789760827.933235:0:2362:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff9811e0606800: Connection to lustre-OST0000 (at 192.168.203.113@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1789760827.933238:0:2362:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00100000:0.0:1789760827.933240:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048669184/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760827.933253:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2362:1876700048666624:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760827.933263:0:2362:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:cfa8bac1-156b-47a9-afab-fe5753451799:2362:1876700048666880:192.168.203.113@tcp:400:kworker.0 00000100:00080000:0.0:1789760827.933271:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.933278:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760827.933281:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760827.933289:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 255 00000100:00080000:0.0:1789760827.933298:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760827.933316:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048669184:192.168.203.113@tcp:38:kworker.0 00000008:00100000:0.0:1789760827.933404:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760827.933433:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048669312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760827.933452:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.933455:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760827.933457:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760827.933481:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 255 00000100:00080000:0.0:1789760827.933490:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760827.933497:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760827.933514:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048669440/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760827.933528:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.933532:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789760827.933535:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669312:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789760827.933546:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760827.933552:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 255 00000100:00080000:0.0:1789760827.933562:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760827.933569:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760827.933587:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048669568/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760827.933590:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669440:192.168.203.113@tcp:8:kworker.0 00000100:00080000:0.0:1789760827.933632:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760827.933637:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1789760827.933646:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669568:192.168.203.113@tcp:8:kworker.0 00010000:00010000:0.0:1789760828.803669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760829.827745:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760830.851659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760831.875627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760832.003630:0:2359:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760816/real 1789760816] req@ffff9811e075ca80 x1876700048667392/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1789760832.003635:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760816/real 1789760816] req@ffff9811e075c000 x1876700048667264/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760832.003746:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811e075c000 x1876700048667264/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760832.003785:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e071a000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:0.0:1789760832.003798:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760816/real 1789760816] req@ffff9811e075f800 x1876700048667520/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760832.003821:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811e075f800 x1876700048667520/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760832.003833:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed5800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1789760832.003861:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048667264:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760832.003872:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048667520:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760832.021562:0:2359:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811e075ca80 x1876700048667392/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760832.021576:0:2359:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e0717000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:1.0:1789760832.021586:0:2359:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760816/real 1789760816] req@ffff9811e075f480 x1876700048667648/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760832.021599:0:2359:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811e075f480 x1876700048667648/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760832 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760832.021606:0:2359:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed2000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:1.0:1789760832.021625:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048667392:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760832.021631:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e075f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048667648:192.168.203.113@tcp:400:kworker.0 00010000:00010000:0.0:1789760832.899715:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789760833.027703:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760833.027754:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760833.027762:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760833.027770:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760833.027780:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760833.027798:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760833.027807:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760833.027841:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760833.027845:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760833.027862:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1789760833.923697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760834.947686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760835.971691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760836.995614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760837.123765:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760827/real 1789760827] req@ffff9811c2e2a300 x1876700048669568/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:1.0:1789760837.123772:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760821/real 1789760821] req@ffff9811c92c5f80 x1876700048668032/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1789760837.123810:0:2359:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760821/real 1789760821] req@ffff9811c92c5880 x1876700048667904/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.123907:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048669568/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760837.123945:0:2359:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c92c5880 x1876700048667904/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.123958:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760827/real 1789760827] req@ffff9811c2e28000 x1876700048669312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760837.123988:0:2359:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e071a000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1789760837.124000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048669312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1789760837.124003:0:2359:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760821/real 1789760821] req@ffff9811c92c4e00 x1876700048668160/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.124046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760827/real 1789760827] req@ffff9811c2e2b800 x1876700048669440/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760837.124056:0:2359:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c92c4e00 x1876700048668160/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760837.124068:0:2359:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed5800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1789760837.124070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048669440/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.124084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760827/real 1789760827] req@ffff9811c2e2b480 x1876700048669184/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760837.124090:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2359:1876700048667904:192.168.203.113@tcp:400:kworker.0 00000100:00100000:0.0:1789760837.124108:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048668160:192.168.203.113@tcp:400:kworker.0 00000100:00100000:2.0:1789760837.124115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048669184/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.124130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760827/real 1789760827] req@ffff9811c2e29500 x1876700048669056/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760837.124145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048669056/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760837.124162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760837.124205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760837.124210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669568:192.168.203.113@tcp:8:kworker.0 00000100:00080000:2.0:1789760837.124217:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760837.124222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760837.124227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669312:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789760837.124253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760837.124260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760837.124267:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669440:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789760837.124271:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760837.124274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760837.124283:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760837.124295:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789760837.124303:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789760837.124305:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048669184:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789760837.124333:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760837.124340:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760837.124354:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 282 00000100:00080000:2.0:1789760837.124355:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760837.124374:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789760837.124399:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789760837.124408:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760837.124409:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048669056:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789760837.124429:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760837.124486:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048669696/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760837.124566:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760837.124571:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760837.124573:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760837.124578:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 282 00000100:00080000:3.0:1789760837.124583:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1789760837.124589:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760837.124596:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760837.124611:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048669824/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760837.124623:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760837.124628:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760837.124632:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760837.124639:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 282 00000100:00100000:0.0:1789760837.124649:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048669696:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789760837.124650:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1789760837.124670:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760837.124681:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760837.124722:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23b80 x1876700048669952/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760837.124742:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760837.124746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760837.124748:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760837.124771:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 282 00000100:00080000:3.0:1789760837.124782:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1789760837.124792:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760837.124803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760837.124808:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669824:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760837.124835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048670080/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760837.124906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669952:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789760837.124908:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760837.124918:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760837.124920:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760837.124926:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 282 00000100:00080000:3.0:1789760837.124934:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1789760837.124939:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760837.124958:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048670208/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760837.124968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670080:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760837.125020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670208:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760837.147967:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c92c5f80 x1876700048668032/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760837.147995:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e0717000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1789760837.148012:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760821/real 1789760821] req@ffff9811c92c4a80 x1876700048668288/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760837.148035:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c92c4a80 x1876700048668288/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760837 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760837.148045:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed2000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760837.148071:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668032:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760837.148080:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668288:192.168.203.113@tcp:400:kworker.0 00010000:00010000:0.0:1789760838.019738:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760838.679013:0:9483:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OST are inactive after seconds, give up 00010000:00010000:0.0:1789760839.043578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760840.067616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760841.091632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760842.115789:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:1.0:1789760842.243624:0:2359:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffff9811c2e29f80 x1876700048668672/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760842.243624:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.243643:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.243657:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.243666:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.243677:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.243689:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.243715:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.243720:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760842.243723:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760842.243726:0:488:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1789760842.266733:0:2359:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c2e29f80 x1876700048668672/t0(0) o400->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760842.266749:0:2359:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e0717000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760842.266769:0:2359:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:cfa8bac1-156b-47a9-afab-fe5753451799:2359:1876700048668672:192.168.203.113@tcp:400:kworker.0 00000100:00000400:1.0:1789760842.307585:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffff9811c2e28a80 x1876700048668928/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760842.307624:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c2e28a80 x1876700048668928/t0(0) o400->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760842.307632:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed2000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1789760842.307639:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffff9811c2e28380 x1876700048668544/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760842.307648:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c2e28380 x1876700048668544/t0(0) o400->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760842.307653:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff9811e071a000 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:1.0:1789760842.307657:0:2360:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760826/real 1789760826] req@ffff9811c2e28e00 x1876700048668800/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760842.307664:0:2360:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9811c2e28e00 x1876700048668800/t0(0) o400->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760842 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760842.307669:0:2360:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff9811c2ed5800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760842.307683:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668928:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760842.307688:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2360:1876700048668544:192.168.203.113@tcp:400:kworker.0 00000100:00100000:1.0:1789760842.307691:0:2360:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:cfa8bac1-156b-47a9-afab-fe5753451799:2360:1876700048668800:192.168.203.113@tcp:400:kworker.0 00010000:00010000:0.0:1789760843.139668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760844.164650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760845.187557:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1789760845.955642:0:2350:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.113@tcp (192.168.203.113:988) timed out; the network or node may be down. 00010000:00010000:0.0:1789760846.211581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760847.235703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760847.363653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760847.363666:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760847.363671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760847.363676:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760847.363681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760847.363686:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760847.363690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760847.363694:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760847.363699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760847.363703:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1789760848.259587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:2.0:1789760849.008148:0:9621:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: OST are inactive after seconds, give up 00010000:00010000:0.0:1789760849.283664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760850.307704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760851.331692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760852.355679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760852.483606:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760837/real 1789760837] req@ffff9811e1a21880 x1876700048670208/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760852.483675:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048670208/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760852.483698:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760837/real 1789760837] req@ffff9811e1a21180 x1876700048670080/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760852.483714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760852.483727:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048670080/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760852.483731:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760852.483791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760852.483794:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760837/real 1789760837] req@ffff9811e1a23b80 x1876700048669952/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760852.483812:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23b80 x1876700048669952/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760852.483823:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760852.483835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.483845:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1789760852.483852:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760837/real 1789760837] req@ffff9811e1a22300 x1876700048669696/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760852.483853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.483869:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1789760852.483881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760852.483888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048669696/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760852.483894:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1789760852.483914:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760837/real 1789760837] req@ffff9811e1a22a00 x1876700048669824/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760852.483928:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048669824/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760852 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760852.483954:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760852.483982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760852.483987:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670208:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789760852.483994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760852.484000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760852.484004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670080:192.168.203.113@tcp:8:kworker.0 00000100:00080000:0.0:1789760852.484011:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760852.484019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760852.484024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669952:192.168.203.113@tcp:8:kworker.0 00000100:00080000:1.0:1789760852.484026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760852.484030:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760852.484036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760852.484043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760852.484050:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789760852.484055:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 291 00000100:00100000:0.0:1789760852.484061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048669696:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789760852.484079:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 10s 00000100:00080000:0.0:1789760852.484083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760852.484097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789760852.484097:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760852.484111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048669824:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760852.484160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048670336/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760852.484224:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.484229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760852.484232:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760852.484237:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 291 00000100:00080000:1.0:1789760852.484242:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1789760852.484247:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760852.484261:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760852.484285:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670336:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760852.484297:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048670464/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760852.484315:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.484319:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760852.484322:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760852.484326:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 291 00000100:00080000:1.0:1789760852.484334:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1789760852.484339:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760852.484346:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760852.484361:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048670592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760852.484373:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.484379:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789760852.484383:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760852.484391:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 291 00000100:00080000:1.0:1789760852.484399:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1789760852.484406:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760852.484417:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760852.484444:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048670720/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760852.484446:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048670464:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760852.484556:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670592:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789760852.484580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760852.484590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789760852.484596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760852.484604:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670720:192.168.203.113@tcp:8:kworker.0 00000100:00080000:1.0:1789760852.484644:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 291 00000100:00080000:1.0:1789760852.484657:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 10s 00000100:00080000:1.0:1789760852.484664:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760852.484681:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760852.484717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048670848/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760852.484850:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670848:192.168.203.113@tcp:8:kworker.0 00010000:00010000:0.0:1789760853.379727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760854.403709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789760854.724037:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:3.0:1789760854.724133:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789760854.724154:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789760854.724205:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789760854.724234:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789760854.724250:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789760854.724267:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789760854.724293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760852/real 1789760854] req@ffff9811c92c4000 x1876700048670720/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760854.724354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048670720/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760854.724373:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760854.724382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760854.724392:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670720:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760854.724410:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760852/real 1789760854] req@ffff9811c92c5f80 x1876700048670464/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760854.724438:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048670464/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760854.724522:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760854.724534:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760854.724545:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048670464:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760854.724561:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760852/real 1789760854] req@ffff9811c92c7480 x1876700048670592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760854.724592:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048670592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760854.724611:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760854.724620:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760854.724628:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760854.724645:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760852/real 1789760854] req@ffff9811c92c4a80 x1876700048670336/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760854.724674:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048670336/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760854.724689:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760854.724695:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760854.724702:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670336:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760854.724781:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760852/real 1789760854] req@ffff9811c92c7800 x1876700048670848/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760854.724805:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048670848/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760854.724834:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760854.724841:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760854.724848:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048670848:192.168.203.113@tcp:8:kworker.0 00010000:00010000:0.0:1789760855.427809:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760856.451582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760857.475678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760857.603626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760857.603639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760857.603644:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760857.603657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 307 00000100:00080000:3.0:1789760857.603672:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1789760857.603699:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760857.603781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048670976/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760857.603808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760857.603812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760857.603814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760857.603817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 307 00000100:00080000:3.0:1789760857.603821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1789760857.603825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760857.603840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760857.603853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048671104/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760857.603865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760857.603884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760857.603886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760857.603892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 307 00000100:00080000:3.0:1789760857.603898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1789760857.603902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760857.603909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760857.603927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048671232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760857.603929:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670976:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760857.603957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760857.603961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760857.603965:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760857.603970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 307 00000100:00080000:3.0:1789760857.603977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1789760857.603981:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760857.603986:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760857.604000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048671360/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760857.604009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760857.604013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760857.604014:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760857.604020:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 307 00000100:00080000:3.0:1789760857.604024:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1789760857.604028:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760857.604040:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760857.604052:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048671488/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760857.604118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048671104:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760857.604173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760857.604208:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671360:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760857.604239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671488:192.168.203.113@tcp:8:kworker.0 00000400:00000100:1.0:1789760857.795979:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:1.0:1789760857.796044:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760857.796071:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760857.796123:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760857.796135:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760857.796145:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760857.796158:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789760857.796212:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760857/real 1789760857] req@ffff9811e1a22680 x1876700048671232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760857.796244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048671232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760857.796275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760857.796284:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760857.796291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760857.796304:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760857/real 1789760857] req@ffff9811e1a22a00 x1876700048671360/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760857.796354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048671360/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760857.796366:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760857.796372:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760857.796377:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671360:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760857.796398:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760857/real 1789760857] req@ffff9811e1a22300 x1876700048671488/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760857.796424:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048671488/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760857.796435:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760857.796440:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760857.796444:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760857.796453:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760857/real 1789760857] req@ffff9811e1a21c00 x1876700048670976/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760857.796544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048670976/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760857.796556:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760857.796560:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760857.796564:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048670976:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760857.796574:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760857/real 1789760857] req@ffff9811e1a22d80 x1876700048671104/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760857.796609:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048671104/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760857.796620:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760857.796624:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760857.796628:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048671104:192.168.203.113@tcp:38:kworker.0 00010000:00010000:0.0:1789760858.499663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760859.523621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760860.547625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760861.571593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760862.595697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760862.723598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760862.723606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760862.723609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760862.723617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 312 00000100:00080000:0.0:1789760862.723626:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760862.723631:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760862.723677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048671616/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760862.723717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760862.723720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760862.723721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760862.723724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 312 00000100:00080000:0.0:1789760862.723727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760862.723729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760862.723751:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760862.723762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048671744/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760862.723768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048671616:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789760862.723775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760862.723779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760862.723780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760862.723784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 312 00000100:00080000:0.0:1789760862.723787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760862.723790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760862.723793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760862.723801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048671872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760862.723808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760862.723811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760862.723812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760862.723814:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 312 00000100:00080000:0.0:1789760862.723816:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760862.723819:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760862.723822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760862.723829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048672000/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760862.723836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760862.723838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789760862.723843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760862.723846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 312 00000100:00080000:0.0:1789760862.723849:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760862.723852:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760862.723856:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760862.723864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048672128/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760862.723874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048671744:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789760862.723900:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789760862.723915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672000:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789760862.723945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672128:192.168.203.113@tcp:8:kworker.0 00010000:00010000:1.0:1789760863.619594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789760863.939962:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:1.0:1789760863.940022:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760863.940055:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760863.940107:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760863.940127:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760863.940138:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760863.940152:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789760863.940203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760862/real 1789760863] req@ffff9811c2e29180 x1876700048672128/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760863.940244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048672128/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760863.940271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760863.940277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760863.940282:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760863.940293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760862/real 1789760863] req@ffff9811c2e2aa00 x1876700048671744/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760863.940317:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048671744/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760863.940329:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760863.940333:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760863.940337:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048671744:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760863.940345:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760862/real 1789760863] req@ffff9811c2e2b100 x1876700048671872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760863.940366:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048671872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760863.940401:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760863.940407:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760863.940412:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048671872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760863.940421:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760862/real 1789760863] req@ffff9811c2e2bb80 x1876700048672000/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760863.940440:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048672000/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760863.940518:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760863.940526:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760863.940532:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672000:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760863.940543:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760862/real 1789760863] req@ffff9811c2e28700 x1876700048671616/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760863.940562:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048671616/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760863.940574:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760863.940579:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760863.940585:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048671616:192.168.203.113@tcp:250:kworker.0 00000001:02000400:2.0:1789760864.597035:0:10182:0:(debug.c:653:libcfs_debug_mark_buffer()) DEBUG MARKER: === sanityn: finish setup 15:47:43 (1789760863) === 00010000:00010000:0.0:1789760864.643612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789760865.667681:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760866.691688:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1789760867.047824:0:7468:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908),name=/ 00000002:00010000:3.0:1789760867.047867:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1789760867.047900:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1789760867.047948:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00000080:00200000:3.0:1789760867.047967:0:7468:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:3.0:1789760867.047983:0:7468:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) 00000080:00010000:3.0:1789760867.047990:0:7468:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff9811d8d18908) for lock 0x3d1f44aaa2c56210 00000080:00010000:3.0:1789760867.048006:0:7468:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x3d1f44aaa2c56210 from it ffffa84e41c4fbb0 00010000:00010000:3.0:1789760867.048016:0:7468:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 8 lru_type: 1 00010000:00010000:3.0:1789760867.048036:0:7468:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000080:00200000:3.0:1789760867.048065:0:7468:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff9811d8d18908), inode mode 41ed mask 44 00000080:00200008:3.0:1789760867.048080:0:7468:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), flags 304000 00000080:00200008:3.0:1789760867.048108:0:7468:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff9811d8d18908), request_mask 2047, flags 0, foreign 0 00000002:00010000:3.0:1789760867.048135:0:7468:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: readdir flags 0 00010000:00010000:3.0:1789760867.048151:0:7468:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00010000:00010000:3.0:1789760867.048169:0:7468:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff9811e1a82800 lock: ffff9811c1519200/0x3d1f44aaa2c56210 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x1287b6a7b7c52903 expref: -99 pid: 5471 timeout: 0 lvb_type: 0 lru_score: 9 lru_type: 1 00000100:00080000:3.0:1789760867.049593:0:7468:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff9811e1a23b80 x1876700048672256/t0(0) o37->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:23/10 lens 448/440 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'bash.0' uid:0 gid:0 projid:0 00000100:00100000:3.0:1789760867.049617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760867.715748:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760867.843701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760867.843713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760867.843719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760867.843732:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 317 00000100:00080000:2.0:1789760867.843746:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760867.843755:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760867.843814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048672384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760867.843856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760867.843861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760867.843864:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760867.843869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 317 00000100:00080000:2.0:1789760867.843873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760867.843877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760867.843907:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760867.843921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048672512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760867.843937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760867.843942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760867.843945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760867.843951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 317 00000100:00080000:2.0:1789760867.843984:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789760867.843995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048672384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760867.843996:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760867.844009:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760867.844024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048672640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760867.844050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760867.844054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760867.844057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760867.844062:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 317 00000100:00080000:2.0:1789760867.844066:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760867.844071:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760867.844077:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760867.844090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048672768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760867.844102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760867.844107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760867.844130:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760867.844135:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 317 00000100:00080000:2.0:1789760867.844140:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760867.844145:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760867.844151:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760867.844164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048672896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760867.844207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048672512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760867.844279:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760867.844309:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760867.844352:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760868.099553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760868.739779:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760869.123511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760869.763706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789760870.084092:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:0.0:1789760870.084149:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789760870.084166:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760870.084209:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760870.084219:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760870.084231:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760870.084245:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789760870.084274:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffff9811d863b480 x1876700048672640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760870.084322:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048672640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760870.084336:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760870.084341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760870.084345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760870.084354:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffff9811d8638000 x1876700048672768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760870.084370:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048672768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760870.084382:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760870.084388:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760870.084417:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760870.084428:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffff9811d863b800 x1876700048672896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760870.084470:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048672896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760870.084558:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760870.084565:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760870.084570:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048672896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760870.084582:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffff9811d8638e00 x1876700048672384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760870.084601:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048672384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760870.084631:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760870.084636:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760870.084641:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048672384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760870.084652:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760867/real 1789760870] req@ffff9811d8638380 x1876700048672512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760870.084670:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048672512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760870.084681:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760870.084686:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760870.084701:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048672512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760870.147568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760870.787760:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760871.171559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760871.811662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760872.195552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760872.835572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760872.963580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760872.963600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760872.963605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760872.963615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 322 00000100:00080000:3.0:1789760872.963627:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760872.963637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760872.963704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048673024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760872.963733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760872.963736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760872.963738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760872.963741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 322 00000100:00080000:3.0:1789760872.963745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760872.963749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760872.963758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760872.963769:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048673024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760872.963771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048673152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760872.963785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760872.963788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760872.963807:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760872.963811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 322 00000100:00080000:3.0:1789760872.963814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760872.963817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760872.963822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760872.963851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048673280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760872.963861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760872.963864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760872.963865:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760872.963872:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 322 00000800:00000100:2.0:1789760872.963873:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760872.963877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760872.963882:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760872.963887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760872.963911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048673408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760872.963918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760872.963920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760872.963922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760872.963925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 322 00000100:00080000:3.0:1789760872.963927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760872.963930:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760872.963934:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760872.963950:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760872/real 1789760872] req@ffff9811e1a21180 x1876700048673024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760872.963951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048673536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760872.964008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048673024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760872.964023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760872.964029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760872.964034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048673024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789760872.964059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048673152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789760872.964080:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760872.964091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760872/real 1789760872] req@ffff9811e1a21880 x1876700048673152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760872.964110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048673152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760872.964116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760872.964119:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760872.964122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048673152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789760872.964126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789760872.964141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760872.964158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760872/real 1789760872] req@ffff9811e1a20700 x1876700048673280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760872.964168:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048673280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760872.964174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760872.964177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760872.964180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789760872.964184:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789760872.964215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760872.964224:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760872/real 1789760872] req@ffff9811e1a23100 x1876700048673408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760872.964234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048673408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760872.964240:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760872.964243:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760872.964245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789760872.964250:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789760872.964263:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760872.964285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760872/real 1789760872] req@ffff9811e1a23480 x1876700048673536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760872.964292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048673536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760872.964302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760872.964305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789760872.964307:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760873.219535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760873.859578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760874.243561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760874.883586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760875.267514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760875.907589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760876.291643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760876.931669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760877.315545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760877.955697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760878.083817:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760878.083829:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760878.083835:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760878.083848:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 327 00000100:00080000:0.0:1789760878.083862:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760878.083873:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760878.083925:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048673664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760878.083981:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760878.083987:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760878.083991:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760878.084000:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 327 00000100:00100000:2.0:1789760878.084012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048673664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789760878.084016:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760878.084032:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760878.084049:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760878.084079:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048673792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760878.084105:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760878.084112:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760878.084116:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760878.084142:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 327 00000100:00080000:0.0:1789760878.084151:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760878.084161:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760878.084170:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760878.084195:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048673920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760878.084209:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760878.084213:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760878.084216:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789760878.084224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048673792:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789760878.084225:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 327 00000100:00080000:0.0:1789760878.084245:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760878.084254:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760878.084267:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760878.084295:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760878.084306:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048674048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760878.084330:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760878.084337:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789760878.084341:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760878.084355:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 327 00000100:00080000:0.0:1789760878.084365:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789760878.084373:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674048:192.168.203.113@tcp:8:kworker.0 00000100:00080000:0.0:1789760878.084377:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789760878.084404:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760878.084439:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048674176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760878.084593:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760878.339576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760878.979636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789760879.299917:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:2.0:1789760879.299970:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760879.299990:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789760879.300043:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789760879.300052:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789760879.300059:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789760879.300068:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789760879.300102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760878/real 1789760879] req@ffff9811c2e28700 x1876700048674176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760879.300137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048674176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760879.300153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760879.300159:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760879.300165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760879.300174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760878/real 1789760879] req@ffff9811c2e2a680 x1876700048674048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760879.300213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048674048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760879.300223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760879.300226:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760879.300230:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760879.300238:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760878/real 1789760879] req@ffff9811c2e29c00 x1876700048673792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760879.300253:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048673792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760879.300263:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760879.300267:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760879.300277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048673792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760879.300286:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760878/real 1789760879] req@ffff9811c2e2ad80 x1876700048673920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760879.300300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048673920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760879.300310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760879.300314:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760879.300317:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048673920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760879.300324:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760878/real 1789760879] req@ffff9811c2e29880 x1876700048673664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760879.300349:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048673664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760879.300358:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760879.300362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760879.300366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048673664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760879.363513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760880.003574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760880.387512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760881.027595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760881.411504:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760882.051612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760882.435502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760883.075598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789760883.203617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760883.203649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760883.203653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760883.203664:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 332 00000100:00080000:1.0:1789760883.203676:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760883.203683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760883.203722:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048674304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760883.203750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760883.203753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760883.203755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760883.203758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 332 00000100:00080000:1.0:1789760883.203761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760883.203764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760883.203773:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760883.203782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048674432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760883.203793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760883.203800:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048674304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789760883.203800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760883.203806:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760883.203819:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 332 00000100:00080000:1.0:1789760883.203823:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760883.203825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760883.203828:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760883.203838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048674560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760883.203850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760883.203853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789760883.203856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760883.203860:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 332 00000100:00080000:1.0:1789760883.203872:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760883.203877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789760883.203886:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789760883.203889:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760883.203904:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048674688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760883.203912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760883.203915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789760883.203917:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760883.203921:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 332 00000100:00080000:1.0:1789760883.203925:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760883.203928:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760883.203932:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760883/real 1789760883] req@ffff9811c92c6300 x1876700048674304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789760883.203932:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760883.203946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048674816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760883.203988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048674304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760883.204000:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760883.204006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760883.204010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048674304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760883.204035:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048674432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760883.204053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760883.204061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760883/real 1789760883] req@ffff9811c92c4700 x1876700048674432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760883.204087:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048674432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760883.204093:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760883.204095:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760883.204098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048674432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760883.204102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760883.204116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760883.204123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760883/real 1789760883] req@ffff9811c92c6a00 x1876700048674560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760883.204130:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048674560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760883.204135:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760883.204136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760883.204138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760883.204142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760883.204152:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760883.204158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760883/real 1789760883] req@ffff9811c92c7100 x1876700048674688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760883.204165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048674688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760883.204178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760883.204180:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760883.204183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760883.204187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760883.204199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760883.204206:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760883/real 1789760883] req@ffff9811c92c7800 x1876700048674816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760883.204213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048674816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760883.204217:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760883.204219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760883.204221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048674816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760883.459502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760884.099622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760884.483498:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760885.123595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760885.507553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760886.147593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760886.531546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760887.171599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760887.555526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760888.195607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760888.323593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760888.323604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760888.323609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760888.323620:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 337 00000100:00080000:2.0:1789760888.323632:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760888.323639:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760888.323698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048674944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760888.323723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760888.323726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760888.323727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760888.323730:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 337 00000100:00080000:2.0:1789760888.323733:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760888.323736:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760888.323744:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760888.323753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048675072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760888.323762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760888.323764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760888.323766:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760888.323769:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 337 00000100:00080000:2.0:1789760888.323771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760888.323774:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760888.323777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760888.323786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048675200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760888.323794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760888.323797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760888.323798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760888.323809:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 337 00000100:00100000:0.0:1789760888.323811:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048674944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760888.323813:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760888.323825:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760888.323830:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760888.323842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048675328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760888.323849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760888.323852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760888.323853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760888.323855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 337 00000100:00080000:2.0:1789760888.323858:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760888.323861:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760888.323872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760888.323881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048675456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760888.323888:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760888.323937:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760888/real 1789760888] req@ffff9811d863a300 x1876700048674944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760888.323952:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048674944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760888.323974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760888.323978:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760888.323981:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048674944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760888.323987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048675072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760888.324005:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760888.324014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760888/real 1789760888] req@ffff9811d8638700 x1876700048675072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760888.324022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048675072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760888.324028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760888.324030:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760888.324032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048675072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760888.324051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760888.324062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760888.324068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760888/real 1789760888] req@ffff9811d863aa00 x1876700048675200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760888.324075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048675200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760888.324081:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760888.324082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760888.324084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760888.324093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760888.324107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760888.324114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760888/real 1789760888] req@ffff9811d8638380 x1876700048675328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760888.324122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048675328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760888.324127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760888.324128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760888.324130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760888.324134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760888.324144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760888.324149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760888/real 1789760888] req@ffff9811d8638e00 x1876700048675456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760888.324167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048675456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760888.324172:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760888.324174:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760888.324176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760888.579525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760889.219618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760889.603610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760890.243583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760890.627569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760891.267615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760891.651584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760892.291578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760892.675515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760893.315612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760893.443616:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760893.443630:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760893.443636:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760893.443649:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 343 00000100:00080000:3.0:1789760893.443663:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760893.443672:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760893.443718:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048675584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760893.443766:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760893.443774:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760893.443776:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760893.443781:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 343 00000100:00080000:3.0:1789760893.443786:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760893.443791:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760893.443803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760893.443817:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048675584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760893.443820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048675712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760893.443838:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760893.443841:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760893.443843:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760893.443848:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 343 00000100:00080000:3.0:1789760893.443858:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760893.443863:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760893.443868:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760893.443881:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048675840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760893.443893:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760893.443896:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760893.443898:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760893.443905:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 343 00000800:00000100:0.0:1789760893.443906:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760893.443916:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760893.443922:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760893.443928:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760893.443939:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048675968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760893.443950:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760893.443954:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760893.443956:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760893.443961:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 343 00000100:00100000:0.0:1789760893.443966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760893/real 1789760893] req@ffff9811e1a20000 x1876700048675584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760893.443967:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760893.443974:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760893.443982:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760893.443995:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048676096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760893.444001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048675584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760893.444021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760893.444025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760893.444028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048675584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760893.444058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048675712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760893.444099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760893.444111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760893/real 1789760893] req@ffff9811e1a23800 x1876700048675712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760893.444120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048675712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760893.444126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760893.444129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760893.444131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048675712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760893.444135:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760893.444147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760893.444153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760893/real 1789760893] req@ffff9811e1a20a80 x1876700048675840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760893.444169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048675840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760893.444175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760893.444177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760893.444179:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760893.444182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760893.444195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760893.444202:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760893/real 1789760893] req@ffff9811e1a20e00 x1876700048675968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760893.444209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048675968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760893.444214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760893.444216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760893.444218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048675968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760893.444221:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760893.444231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760893.444237:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760893/real 1789760893] req@ffff9811e1a20380 x1876700048676096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760893.444244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048676096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760893.444258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760893.444260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760893.444263:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760893.699592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760894.339611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760894.723515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760895.363644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760895.747586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760896.387606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760896.771517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760897.411565:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760897.795573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760898.435613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760898.563536:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760898.563548:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760898.563552:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760898.563563:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 348 00000100:00080000:3.0:1789760898.563576:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760898.563584:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760898.563648:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048676224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760898.563676:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760898.563679:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760898.563680:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760898.563683:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 348 00000100:00080000:3.0:1789760898.563686:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760898.563689:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760898.563699:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760898.563708:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048676352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760898.563718:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760898.563723:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048676224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760898.563724:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760898.563727:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760898.563736:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 348 00000100:00080000:3.0:1789760898.563738:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760898.563740:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760898.563743:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760898.563763:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048676480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760898.563770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760898.563772:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760898.563774:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760898.563783:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 348 00000100:00080000:3.0:1789760898.563786:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760898.563789:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760898.563794:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760898.563801:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048676608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760898.563808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760898.563811:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760898.563812:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760898.563814:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 348 00000100:00080000:3.0:1789760898.563817:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760898.563819:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760898.563822:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760898.563830:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048676736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760898.563858:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048676352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760898.563885:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760898.563908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760898.563925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760898.819544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760899.459585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760899.843519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760900.483575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789760900.804013:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.203.113:988 00000400:00000100:1.0:1789760900.804091:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760900.804115:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760900.804183:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760900.804235:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760900.804243:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760900.804251:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789760900.804333:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760898/real 1789760900] req@ffff9811e1a21500 x1876700048676352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760900.804428:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048676352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760900.804453:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760900.804529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760900.804568:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048676352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760900.804589:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760898/real 1789760900] req@ffff9811e1a20380 x1876700048676480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760900.804621:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048676480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760900.804638:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760900.804645:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760900.804652:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760900.804665:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760898/real 1789760900] req@ffff9811e1a20e00 x1876700048676608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760900.804688:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048676608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760900.804705:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760900.804712:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760900.804720:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760900.804734:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760898/real 1789760900] req@ffff9811e1a20a80 x1876700048676736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760900.804758:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048676736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760900.804773:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760900.804780:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760900.804800:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048676736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760900.804816:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760898/real 1789760900] req@ffff9811e1a21f80 x1876700048676224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760900.804846:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048676224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760900.804889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760900.804897:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760900.804905:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048676224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760900.867532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760901.507581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760901.891517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760902.531612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760902.915507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760903.555718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789760903.683569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760903.683581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760903.683585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760903.683597:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 353 00000100:00080000:1.0:1789760903.683611:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760903.683619:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760903.683683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048676864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760903.683718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760903.683722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760903.683724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760903.683728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 353 00000100:00080000:1.0:1789760903.683731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760903.683734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760903.683745:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760903.683756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048676992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760903.683766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760903.683769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760903.683771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760903.683775:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 353 00000100:00100000:0.0:1789760903.683807:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048676864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789760903.683809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760903.683816:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760903.683821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760903.683830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048677120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760903.683838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760903.683841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789760903.683842:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760903.683845:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 353 00000100:00080000:1.0:1789760903.683848:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760903.683850:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760903.683853:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760903.683875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048677248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760903.683885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760903.683889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789760903.683891:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760903.683895:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 353 00000800:00000100:0.0:1789760903.683901:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789760903.683902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760903.683910:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760903.683917:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760903.683930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048677376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760903.683957:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760903/real 1789760903] req@ffff9811c92c4a80 x1876700048676864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760903.683975:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048676864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760903.683985:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760903.683989:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760903.683992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048676864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760903.683997:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048676992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760903.684011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760903.684029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760903/real 1789760903] req@ffff9811c92c7480 x1876700048676992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760903.684038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048676992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760903.684044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760903.684046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760903.684049:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048676992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760903.684074:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760903.684090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760903.684101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760903/real 1789760903] req@ffff9811c92c5f80 x1876700048677120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760903.684114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048677120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760903.684123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760903.684127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760903.684130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760903.684137:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760903.684157:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760903.684166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760903/real 1789760903] req@ffff9811c92c4000 x1876700048677248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760903.684174:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048677248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760903.684185:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760903.684188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760903.684190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760903.684195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760903.684209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760903.684228:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760903/real 1789760903] req@ffff9811c92c4e00 x1876700048677376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760903.684235:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048677376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760903.684240:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760903.684242:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760903.684244:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760903.939519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760904.579675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760904.963526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760905.603644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760905.987529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760906.627669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760907.011557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760907.651586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760908.035599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760908.675643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760908.803637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760908.803649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760908.803655:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760908.803668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 358 00000100:00080000:2.0:1789760908.803700:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760908.803709:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760908.803753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048677504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760908.803813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760908.803834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760908.803837:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760908.803843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 358 00000100:00080000:2.0:1789760908.803848:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760908.803853:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760908.803865:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760908.803882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048677632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760908.803899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760908.803903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760908.803906:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760908.803911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 358 00000100:00080000:2.0:1789760908.803915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760908.803919:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760908.803924:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760908.803937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048677760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760908.803955:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048677504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760908.803957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760908.803972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760908.803979:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760908.803986:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 358 00000100:00080000:2.0:1789760908.803989:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760908.803997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760908.804015:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760908.804031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048677888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760908.804043:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760908.804047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760908.804048:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760908.804052:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 358 00000100:00080000:2.0:1789760908.804058:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760908.804063:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760908.804067:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760908.804078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048678016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760908.804094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760908.804176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760908/real 1789760908] req@ffff9811d863b800 x1876700048677504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760908.804197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048677504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760908.804223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760908.804227:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760908.804230:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048677504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760908.804237:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048677632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760908.804273:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760908.804283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760908/real 1789760908] req@ffff9811d8638000 x1876700048677632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760908.804292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048677632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760908.804299:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760908.804301:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760908.804303:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048677632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760908.804365:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760908.804389:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760908.804403:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760908/real 1789760908] req@ffff9811d863b480 x1876700048677760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760908.804423:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048677760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760908.804429:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760908.804432:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760908.804434:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760908.804446:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760908.804520:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760908.804530:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760908/real 1789760908] req@ffff9811d8638a80 x1876700048677888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760908.804539:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048677888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760908.804546:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760908.804556:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760908.804559:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048677888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760908.804564:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760908.804578:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760908.804585:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760908/real 1789760908] req@ffff9811d8639f80 x1876700048678016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760908.804593:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048678016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760908.804598:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760908.804599:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760908.804601:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760909.059538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760909.699586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760910.083511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760910.723574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760911.107502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760911.747677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760912.131523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760912.771585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760913.155536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760913.795653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760913.923584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760913.923598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760913.923603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760913.923616:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 363 00000100:00080000:2.0:1789760913.923630:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760913.923640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760913.923699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048678144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760913.923729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760913.923733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760913.923735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760913.923739:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 363 00000100:00080000:2.0:1789760913.923767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760913.923771:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048678144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760913.923776:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760913.923789:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760913.923807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048678272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760913.923821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760913.923825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760913.923828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760913.923832:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 363 00000100:00080000:2.0:1789760913.923837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760913.923841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760913.923847:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760913.923860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048678400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760913.923866:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789760913.923879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760913.923884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760913.923886:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760913.923891:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 363 00000100:00080000:2.0:1789760913.923895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760913.923899:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760913.923904:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760913.923914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048678528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760913.923927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760913/real 1789760913] req@ffff9811d8639500 x1876700048678144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760913.923934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760913.923937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760913.923939:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760913.923946:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 363 00000100:00080000:2.0:1789760913.923950:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760913.923954:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760913.923959:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760913.923963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048678144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760913.923969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048678656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760913.923978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760913.923984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760913.923988:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048678144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760913.924020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048678272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760913.924043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760913.924056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760913/real 1789760913] req@ffff9811d8639880 x1876700048678272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760913.924075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048678272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760913.924082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760913.924085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760913.924087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048678272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760913.924092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760913.924109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760913.924116:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760913/real 1789760913] req@ffff9811d863a680 x1876700048678400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760913.924141:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048678400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760913.924147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760913.924151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760913.924154:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760913.924159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760913.924181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760913.924191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760913/real 1789760913] req@ffff9811d863ad80 x1876700048678528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760913.924201:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048678528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760913.924207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760913.924210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760913.924212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760913.924216:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760913.924227:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760913.924234:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760913/real 1789760913] req@ffff9811d8639c00 x1876700048678656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760913.924242:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048678656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760913.924258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760913.924262:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760913.924266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048678656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760914.179529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760914.819575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760915.203577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760915.843623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760916.227509:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760916.867614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760917.251540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760917.891587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760918.275571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760918.915615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760919.043591:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760919.043602:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760919.043606:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760919.043618:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 368 00000100:00080000:3.0:1789760919.043629:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760919.043637:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760919.043671:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048678784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760919.043697:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760919.043700:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760919.043701:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760919.043704:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 368 00000100:00080000:3.0:1789760919.043707:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760919.043710:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760919.043718:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760919.043727:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048678912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760919.043768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048678784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760919.043791:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760919.043794:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760919.043796:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760919.043800:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 368 00000100:00080000:3.0:1789760919.043803:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760919.043805:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760919.043809:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760919.043820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048679040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760919.043827:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760919.043829:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760919.043830:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760919.043832:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 368 00000100:00080000:3.0:1789760919.043835:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760919.043837:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760919.043840:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760919.043846:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048679168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760919.043852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760919.043858:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760919.043861:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760919.043862:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760919.043866:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 368 00000100:00080000:3.0:1789760919.043873:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760919.043877:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760919.043880:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760919.043915:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760919/real 1789760919] req@ffff9811e1a23800 x1876700048678784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760919.043916:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048679296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760919.043967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048678784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760919.043981:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760919.043985:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760919.043988:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048678784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760919.044035:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048678912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760919.044055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760919.044076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760919/real 1789760919] req@ffff9811e1a20000 x1876700048678912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760919.044086:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048678912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760919.044119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760919.044121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760919.044124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048678912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760919.044128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760919.044141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760919.044148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760919/real 1789760919] req@ffff9811e1a23480 x1876700048679040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760919.044156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048679040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760919.044161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760919.044163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760919.044165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760919.044168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760919.044178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760919.044184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760919/real 1789760919] req@ffff9811e1a23100 x1876700048679168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760919.044190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048679168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760919.044195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760919.044197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760919.044203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760919.044207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760919.044219:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760919.044226:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760919/real 1789760919] req@ffff9811e1a20700 x1876700048679296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760919.044233:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048679296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760919.044238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760919.044239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760919.044241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760919.299548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760919.939591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760920.323524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760920.963590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760921.347583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760921.987694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760922.371534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760923.011670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760923.395540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760924.035678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789760924.163745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760924.163761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760924.163768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760924.163785:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 373 00000100:00080000:1.0:1789760924.163807:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760924.163822:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760924.163900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048679424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760924.163957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760924.163967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760924.163972:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760924.163981:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 373 00000100:00080000:1.0:1789760924.164004:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760924.164013:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760924.164020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048679424:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789760924.164033:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760924.164062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048679552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760924.164085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789760924.164088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789760924.164115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760924.164143:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760924.164145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760924/real 1789760924] req@ffff9811c92c5880 x1876700048679424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760924.164163:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 373 00000100:00080000:1.0:1789760924.164185:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760924.164193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048679424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760924.164200:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789760924.164242:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760924.164265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789760924.164268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760924.164292:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048679680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760924.164294:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048679424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789760924.164324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760924.164329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789760924.164332:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760924.164338:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 373 00000100:00080000:1.0:1789760924.164342:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760924.164350:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048679552:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789760924.164359:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760924.164370:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760924.164385:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048679808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760924.164400:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789760924.164401:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760924.164413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789760924.164417:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760924.164424:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 373 00000100:00080000:1.0:1789760924.164429:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760924.164432:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760924/real 1789760924] req@ffff9811c92c5500 x1876700048679552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760924.164437:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760924.164454:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760924.164462:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048679552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760924.164519:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760924.164525:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760924.164532:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048679552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760924.164552:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760924.164587:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760924.164603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048679936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760924.164637:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760924/real 1789760924] req@ffff9811c92c4380 x1876700048679680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760924.164664:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048679680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760924.164674:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760924.164680:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760924.164683:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760924.164704:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760924.164726:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760924.164743:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760924/real 1789760924] req@ffff9811c92c6680 x1876700048679808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760924.164752:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048679808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760924.164757:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760924.164759:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760924.164761:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760924.164765:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760924.164777:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760924.164783:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760924/real 1789760924] req@ffff9811c92c5c00 x1876700048679936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760924.164790:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048679936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760924.164795:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760924.164797:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760924.164799:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048679936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760924.419552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760925.059603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760925.443576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760926.083591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760926.467562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760927.107623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760927.491556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760928.131590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760928.515582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760929.155555:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760929.283556:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760929.283567:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760929.283571:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760929.283583:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 378 00000100:00080000:3.0:1789760929.283594:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760929.283602:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760929.283640:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048680064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760929.283680:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760929.283699:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760929.283702:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760929.283707:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 378 00000100:00080000:3.0:1789760929.283712:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760929.283718:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760929.283730:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048680064:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789760929.283730:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760929.283751:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048680192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760929.283767:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760929.283771:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760929.283773:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760929.283777:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 378 00000100:00080000:3.0:1789760929.283782:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760929.283788:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760929.283793:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789760929.283797:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760929.283811:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048680320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760929.283823:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760929.283826:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760929.283828:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760929.283833:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 378 00000100:00080000:3.0:1789760929.283838:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760929.283843:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760929/real 1789760929] req@ffff9811e1a21880 x1876700048680064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760929.283845:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760929.283860:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760929.283877:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048680448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760929.283888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048680064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760929.283891:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760929.283901:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760929.283905:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760929.283912:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760929.283912:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 378 00000100:00080000:0.0:1789760929.283922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789760929.283922:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760929.283928:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048680064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760929.283941:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760929.283947:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789760929.283955:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048680192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760929.283963:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048680576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760929.283986:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760929.284004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760929/real 1789760929] req@ffff9811e1a21180 x1876700048680192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760929.284019:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048680192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760929.284027:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760929.284031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760929.284034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048680192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760929.284041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760929.284060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760929.284072:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760929/real 1789760929] req@ffff9811e1a22d80 x1876700048680320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760929.284085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048680320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760929.284106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760929.284110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760929.284114:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760929.284121:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760929.284144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760929.284155:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760929/real 1789760929] req@ffff9811e1a21c00 x1876700048680448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760929.284180:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048680448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760929.284188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760929.284192:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760929.284195:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760929.284219:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760929.284239:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760929.284250:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760929/real 1789760929] req@ffff9811e1a22300 x1876700048680576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760929.284262:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048680576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760929.284270:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760929.284272:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760929.284282:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760929.539570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760930.179591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760930.563570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760931.203585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760931.587547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760932.227611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760932.611544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760933.251644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760933.635525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760934.275554:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760934.403515:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760934.403523:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760934.403526:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760934.403534:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 383 00000100:00080000:3.0:1789760934.403544:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760934.403549:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760934.403592:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048680704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760934.403614:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760934.403617:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760934.403618:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760934.403621:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 383 00000100:00080000:3.0:1789760934.403624:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760934.403626:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760934.403634:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760934.403642:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048680832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760934.403650:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760934.403655:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789760934.403657:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048680704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760934.403657:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760934.403665:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 383 00000100:00080000:3.0:1789760934.403668:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760934.403670:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760934.403673:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760934.403681:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048680960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760934.403688:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760934.403691:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760934.403692:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760934.403700:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 383 00000100:00080000:3.0:1789760934.403704:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760934.403706:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760934.403711:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760934.403724:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048681088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760934.403734:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760934.403737:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760934.403740:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760934.403745:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 383 00000100:00080000:3.0:1789760934.403749:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760934.403755:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048680832:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789760934.403756:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760934.403766:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760934.403780:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048681216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760934.403786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760934.403809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760934.403824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681216:192.168.203.113@tcp:8:kworker.0 00000400:00000100:1.0:1789760934.595838:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789760934.595917:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760934.595951:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760934.596006:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760934.596020:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760934.596033:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789760934.596046:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789760934.596098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760934/real 1789760934] req@ffff9811e1a22d80 x1876700048681216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760934.596168:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048681216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760934.596185:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760934.596193:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760934.596197:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760934.596209:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760934/real 1789760934] req@ffff9811e1a22680 x1876700048680832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760934.596225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048680832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760934.596234:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760934.596238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760934.596242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048680832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760934.596273:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760934/real 1789760934] req@ffff9811e1a22300 x1876700048680960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760934.596289:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048680960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760934.596298:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760934.596301:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760934.596304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048680960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760934.596311:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760934/real 1789760934] req@ffff9811e1a21c00 x1876700048681088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760934.596322:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048681088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760934.596330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760934.596334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760934.596337:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760934.596360:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760934/real 1789760934] req@ffff9811e1a22a00 x1876700048680704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760934.596374:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048680704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760934.596382:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760934.596386:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760934.596388:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048680704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760934.659548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760935.299579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760935.683518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760936.323628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760936.707518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760937.347609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760937.731512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760938.371599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760938.755529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760939.395635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760939.523568:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760939.523576:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760939.523595:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760939.523603:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 389 00000100:00080000:3.0:1789760939.523612:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760939.523617:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760939.523647:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048681344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760939.523671:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760939.523674:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760939.523675:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760939.523678:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 389 00000100:00080000:3.0:1789760939.523680:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760939.523682:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760939.523689:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760939.523698:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048681472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760939.523706:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760939.523709:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760939.523710:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760939.523713:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 389 00000100:00080000:3.0:1789760939.523715:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760939.523719:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048681344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760939.523720:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760939.523727:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760939.523734:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048681600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760939.523751:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760939.523754:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760939.523755:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760939.523758:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 389 00000100:00080000:3.0:1789760939.523761:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760939.523764:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760939.523768:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789760939.523778:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760939.523779:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048681728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760939.523788:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760939.523791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760939.523792:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760939.523795:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 389 00000100:00080000:3.0:1789760939.523798:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760939.523800:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760939.523803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760939.523810:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048681856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760939.523817:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760939/real 1789760939] req@ffff9811e1a21180 x1876700048681344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760939.523835:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048681344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760939.523844:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760939.523847:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760939.523849:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048681344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760939.523855:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048681472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760939.523870:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760939.523878:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760939/real 1789760939] req@ffff9811e1a21880 x1876700048681472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760939.523885:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048681472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760939.523891:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760939.523893:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760939.523895:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048681472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760939.523912:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760939.523923:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760939.523929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760939/real 1789760939] req@ffff9811e1a20700 x1876700048681600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760939.523944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048681600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760939.523950:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760939.523952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760939.523954:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760939.523958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760939.523970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760939.523976:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760939/real 1789760939] req@ffff9811e1a23100 x1876700048681728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760939.523983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048681728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760939.523989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760939.523990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760939.523992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760939.523995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760939.524005:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760939.524010:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760939/real 1789760939] req@ffff9811e1a23480 x1876700048681856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760939.524017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048681856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760939.524032:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760939.524035:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760939.524037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048681856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760939.779516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760940.419572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760940.803529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760941.443577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760941.827535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760942.467633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760942.851557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760943.491598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760943.875551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760944.515638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789760944.643615:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760944.643626:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789760944.643630:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760944.643642:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 394 00000100:00080000:1.0:1789760944.643656:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760944.643665:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760944.643711:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048681984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760944.643739:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760944.643742:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760944.643763:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760944.643768:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 394 00000100:00080000:1.0:1789760944.643771:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760944.643775:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789760944.643786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048681984:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789760944.643787:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760944.643808:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048682112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760944.643823:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760944.643828:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789760944.643830:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760944.643835:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 394 00000100:00080000:1.0:1789760944.643850:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760944.643854:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760944.643859:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789760944.643862:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760944.643873:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048682240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760944.643882:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760944.643884:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789760944.643886:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760944.643889:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 394 00000100:00080000:1.0:1789760944.643893:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760944.643895:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760944.643900:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760944.643917:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048682368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760944.643920:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760944/real 1789760944] req@ffff9811c92c6d80 x1876700048681984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760944.643933:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789760944.643938:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789760944.643939:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789760944.643942:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 394 00000100:00080000:1.0:1789760944.643945:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760944.643949:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048681984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760944.643952:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789760944.643958:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789760944.643969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789760944.643972:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048682496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760944.643975:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760944.643982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048681984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789760944.644010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048682112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760944.644030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760944.644044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760944/real 1789760944] req@ffff9811c92c5180 x1876700048682112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760944.644072:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048682112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760944.644081:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760944.644085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760944.644098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048682112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760944.644113:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789760944.644139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760944.644152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760944/real 1789760944] req@ffff9811c92c7b80 x1876700048682240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760944.644161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048682240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760944.644167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760944.644170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760944.644172:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789760944.644176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760944.644192:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760944.644198:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760944/real 1789760944] req@ffff9811c92c5c00 x1876700048682368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760944.644206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048682368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760944.644211:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760944.644213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760944.644215:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789760944.644218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789760944.644228:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789760944.644242:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760944/real 1789760944] req@ffff9811c92c6680 x1876700048682496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760944.644251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048682496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760944.644256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760944.644258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789760944.644260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760944.899547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760945.539632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760945.923587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760946.563607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760946.947560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760947.587663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760947.971578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760948.611613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760948.995545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760949.635645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760949.763668:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760949.763679:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760949.763685:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760949.763696:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 399 00000100:00080000:3.0:1789760949.763709:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760949.763717:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760949.763761:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048682624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760949.763796:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760949.763800:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760949.763801:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760949.763804:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 399 00000100:00080000:3.0:1789760949.763807:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760949.763810:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760949.763820:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760949.763831:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048682752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760949.763842:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789760949.763844:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048682624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789760949.763845:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760949.763856:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760949.763863:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 399 00000100:00080000:3.0:1789760949.763870:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760949.763875:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760949.763883:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760949.763894:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048682880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760949.763903:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760949.763906:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760949.763909:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760949.763914:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 399 00000100:00080000:3.0:1789760949.763917:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760949.763922:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789760949.763928:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789760949.763931:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760949.763943:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048683008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760949.763952:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760949.763955:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760949.763957:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789760949.763975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760949/real 1789760949] req@ffff9811e1a20000 x1876700048682624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760949.763991:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048682624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760949.764000:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760949.764003:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760949.764006:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048682624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760949.764045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048682752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760949.764077:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760949.764092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760949/real 1789760949] req@ffff9811e1a23800 x1876700048682752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760949.764114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048682752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760949.764123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760949.764125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760949.764128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048682752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760949.764133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760949.764152:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760949.764160:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760949/real 1789760949] req@ffff9811e1a21f80 x1876700048682880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760949.764170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048682880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760949.764176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760949.764178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760949.764181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048682880:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789760949.764185:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 399 00000100:00100000:1.0:1789760949.764187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683008:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789760949.764191:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760949.764197:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760949.764205:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789760949.764209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760949.764222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760949/real 1789760949] req@ffff9811e1a20a80 x1876700048683008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760949.764232:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048683136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760949.764280:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048683008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760949.764294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760949.764302:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760949.764309:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760949.764332:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760949.764355:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760949.764367:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760949/real 1789760949] req@ffff9811e1a20e00 x1876700048683136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760949.764378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048683136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760949.764384:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760949.764386:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760949.764388:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760950.019540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760950.659596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760951.043540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760951.683605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760952.067591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760952.707651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760953.091530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760953.731593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760954.115648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760954.755594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760954.883559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760954.883569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760954.883574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760954.883596:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 404 00000100:00080000:2.0:1789760954.883610:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760954.883620:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760954.883680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048683264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760954.883708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760954.883711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760954.883713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760954.883720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 404 00000100:00080000:2.0:1789760954.883723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760954.883727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760954.883750:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760954.883753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048683264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789760954.883764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048683392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760954.883776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760954.883779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760954.883781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760954.883784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 404 00000100:00080000:2.0:1789760954.883787:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760954.883790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760954.883794:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760954.883803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048683520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760954.883810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760954.883814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760954.883815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760954.883818:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 404 00000100:00080000:2.0:1789760954.883820:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760954.883823:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760954.883826:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789760954.883828:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760954.883838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048683648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760954.883856:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760954.883859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760954.883865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760954.883869:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 404 00000100:00080000:2.0:1789760954.883873:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789760954.883876:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760954/real 1789760954] req@ffff9811d8639180 x1876700048683264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760954.883879:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760954.883886:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760954.883898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048683776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760954.883953:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048683264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760954.883967:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760954.883973:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760954.883987:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048683264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760954.884017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048683392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760954.884046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760954.884062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760954/real 1789760954] req@ffff9811d863bb80 x1876700048683392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760954.884074:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048683392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760954.884080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760954.884083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760954.884085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048683392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760954.884090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760954.884107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760954.884114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760954/real 1789760954] req@ffff9811d863b100 x1876700048683520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760954.884122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048683520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760954.884137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760954.884140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760954.884142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760954.884146:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760954.884162:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760954.884169:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760954/real 1789760954] req@ffff9811d8639c00 x1876700048683648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760954.884178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048683648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760954.884186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760954.884189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760954.884193:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760954.884199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760954.884221:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760954.884233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760954/real 1789760954] req@ffff9811d863ad80 x1876700048683776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760954.884246:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048683776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760954.884254:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760954.884258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760954.884269:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048683776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760955.139524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760955.779677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760956.163634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760956.803655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760957.187549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760957.827608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760958.211592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760958.851603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760959.235562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760959.875654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760960.003596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760960.003613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760960.003620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760960.003635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 409 00000100:00080000:3.0:1789760960.003652:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760960.003675:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760960.003725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048683904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760960.003754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760960.003757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760960.003759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760960.003763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 409 00000100:00080000:3.0:1789760960.003767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760960.003770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760960.003781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760960.003809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048683904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760960.003817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048684032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760960.003831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760960.003835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760960.003837:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760960.003841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 409 00000100:00080000:3.0:1789760960.003844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760960.003848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760960.003853:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760960.003898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048684160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760960.003916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789760960.003924:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760960.003927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760960.003934:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760960.003970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 409 00000100:00080000:3.0:1789760960.003993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760960.004000:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760960.004009:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760960.004028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048684288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760960.004037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760960.004040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760960.004043:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760960.004058:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 409 00000100:00100000:1.0:1789760960.004059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760960/real 1789760960] req@ffff9811e1a20380 x1876700048683904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760960.004063:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760960.004074:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760960.004082:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789760960.004091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048683904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760960.004098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048684416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789760960.004113:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760960.004121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760960.004126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048683904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760960.004154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048684032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760960.004186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760960.004198:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760960/real 1789760960] req@ffff9811e1a21500 x1876700048684032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760960.004209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048684032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760960.004216:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760960.004219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760960.004222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048684032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760960.004227:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760960.004283:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760960.004296:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760960/real 1789760960] req@ffff9811e1a20e00 x1876700048684160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760960.004309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048684160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760960.004317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760960.004320:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760960.004323:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760960.004329:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760960.004347:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760960.004356:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760960/real 1789760960] req@ffff9811e1a20a80 x1876700048684288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760960.004367:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048684288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760960.004375:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760960.004378:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760960.004380:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760960.004386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760960.004400:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760960.004408:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760960/real 1789760960] req@ffff9811e1a21f80 x1876700048684416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760960.004436:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048684416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760960.004444:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760960.004447:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760960.004450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760960.259556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760960.899584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760961.283551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760961.923713:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760962.307545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760962.947713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760963.331591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760963.971645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760964.355563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760964.995670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760965.123563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760965.123571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760965.123575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760965.123583:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 414 00000100:00080000:3.0:1789760965.123593:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760965.123598:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760965.123628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048684544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760965.123657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760965.123662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760965.123664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760965.123668:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 414 00000100:00080000:3.0:1789760965.123672:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760965.123676:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760965.123701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760965.123712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048684672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760965.123721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760965.123724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760965.123725:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760965.123728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 414 00000100:00080000:3.0:1789760965.123731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760965.123734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760965.123737:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760965.123743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048684544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760965.123746:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048684800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760965.123757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760965.123759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760965.123760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760965.123764:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 414 00000100:00080000:3.0:1789760965.123766:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760965.123768:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760965.123771:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760965.123780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048684928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760965.123786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760965.123788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760965.123794:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760965.123797:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 414 00000100:00080000:3.0:1789760965.123800:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789760965.123805:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760965.123805:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760965.123812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760965.123822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048685056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760965.123861:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760965/real 1789760965] req@ffff9811e1a23800 x1876700048684544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760965.123876:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048684544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760965.123885:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760965.123889:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760965.123892:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048684544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760965.123911:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048684672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760965.123931:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760965.123942:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760965/real 1789760965] req@ffff9811e1a20000 x1876700048684672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760965.123972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048684672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760965.123981:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760965.123985:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760965.123989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048684672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760965.124012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760965.124037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760965.124046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760965/real 1789760965] req@ffff9811e1a23480 x1876700048684800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760965.124054:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048684800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760965.124059:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760965.124061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760965.124079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760965.124084:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760965.124099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760965.124106:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760965/real 1789760965] req@ffff9811e1a23100 x1876700048684928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760965.124115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048684928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760965.124133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760965.124136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760965.124138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048684928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760965.124143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760965.124159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760965.124166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760965/real 1789760965] req@ffff9811e1a20700 x1876700048685056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760965.124175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048685056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760965.124181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760965.124183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760965.124185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760965.379605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760966.019590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760966.403541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760967.043602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760967.427601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760968.067683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760968.451571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760969.091641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760969.475587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760970.115637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760970.243537:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760970.243547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760970.243551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760970.243561:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 419 00000100:00080000:3.0:1789760970.243572:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760970.243580:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760970.243616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048685184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760970.243638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760970.243641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760970.243642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760970.243645:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 419 00000100:00080000:3.0:1789760970.243648:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760970.243664:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760970.243676:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760970.243687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048685312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760970.243697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760970.243700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760970.243702:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760970.243704:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 419 00000100:00080000:3.0:1789760970.243708:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760970.243710:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760970.243714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760970.243717:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048685184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760970.243724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048685440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760970.243732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760970.243735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760970.243736:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760970.243738:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 419 00000100:00080000:3.0:1789760970.243740:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760970.243743:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760970.243746:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760970.243753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048685568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760970.243759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760970.243765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760970.243767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760970.243770:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 419 00000800:00000100:1.0:1789760970.243776:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760970.243780:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760970.243785:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760970.243790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760970.243800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048685696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760970.243826:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760970/real 1789760970] req@ffff9811e1a21880 x1876700048685184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760970.243847:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048685184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760970.243887:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760970.243890:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760970.243893:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048685184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789760970.243899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048685312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760970.243918:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760970.243928:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760970/real 1789760970] req@ffff9811e1a21180 x1876700048685312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760970.243936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048685312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760970.243942:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760970.243944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760970.243946:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048685312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760970.243963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760970.243974:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760970.243980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760970/real 1789760970] req@ffff9811e1a22a00 x1876700048685440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760970.243987:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048685440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760970.243992:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760970.243993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760970.243995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760970.244005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760970.244017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760970.244024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760970/real 1789760970] req@ffff9811e1a21c00 x1876700048685568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760970.244031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048685568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760970.244036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760970.244038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760970.244040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760970.244043:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760970.244053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760970.244058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760970/real 1789760970] req@ffff9811e1a22300 x1876700048685696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760970.244065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048685696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760970.244069:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760970.244071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760970.244073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048685696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760970.499556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760971.139595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760971.523523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760972.163606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760972.547546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760973.187633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760973.571631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760974.211609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760974.595543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760975.235776:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760975.363591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760975.363604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760975.363609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760975.363624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 424 00000100:00080000:3.0:1789760975.363639:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760975.363648:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760975.363700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048685824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760975.363754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760975.363758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760975.363761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760975.363766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 424 00000100:00080000:3.0:1789760975.363808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760975.363813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760975.363839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760975.363850:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048685824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760975.363861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048685952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760975.363876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760975.363911:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760975.363914:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760975.363920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 424 00000100:00080000:3.0:1789760975.363928:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760975.363935:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760975.363946:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789760975.363954:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760975.363975:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048686080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760975.363994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789760975.363997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760975/real 1789760975] req@ffff9811e1a22680 x1876700048685824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760975.364011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760975.364021:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760975.364030:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 424 00000100:00100000:1.0:1789760975.364036:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048685824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760975.364040:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760975.364058:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760975.364066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789760975.364066:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760975.364077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048685824:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789760975.364080:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760975.364099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048686208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760975.364108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048685952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760975.364137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789760975.364138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760975.364146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760975.364149:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760975.364156:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 424 00000100:00100000:1.0:1789760975.364160:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760975/real 1789760975] req@ffff9811e1a22d80 x1876700048685952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760975.364165:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789760975.364188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048685952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760975.364195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760975.364211:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789760975.364212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760975.364225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760975.364228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048685952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760975.364234:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048686336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760975.364248:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760975.364294:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760975.364319:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760975/real 1789760975] req@ffff9811e1a22300 x1876700048686080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760975.364347:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048686080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760975.364353:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760975.364355:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760975.364358:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760975.364371:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760975.364389:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760975.364397:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760975/real 1789760975] req@ffff9811e1a21c00 x1876700048686208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760975.364404:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048686208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760975.364409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760975.364411:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760975.364413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760975.364418:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760975.364428:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760975.364434:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760975/real 1789760975] req@ffff9811e1a22a00 x1876700048686336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760975.364441:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048686336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760975.364455:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760975.364458:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760975.364500:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760975.619701:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760976.259666:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760976.643570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760977.283720:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760977.667564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760978.307611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760978.691563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760979.331671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760979.715559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760980.355685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760980.483517:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760980.483531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760980.483537:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760980.483550:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 430 00000100:00080000:3.0:1789760980.483564:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760980.483573:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760980.483625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048686464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760980.483663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760980.483668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760980.483713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760980.483722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 430 00000100:00080000:3.0:1789760980.483731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760980.483739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760980.483754:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789760980.483760:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048686464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760980.483803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048686592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760980.483847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760980.483853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760980.483856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760980.483862:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 430 00000100:00080000:3.0:1789760980.483869:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760980.483876:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789760980.483884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789760980.483894:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760980.483911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048686720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760980.483929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760980.483935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760980.483938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760980.483945:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 430 00000100:00080000:3.0:1789760980.483953:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760980.483961:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760980.483969:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760980/real 1789760980] req@ffff9811e1a21180 x1876700048686464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789760980.483973:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760980.484006:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048686848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760980.484015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048686464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760980.484023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760980.484034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789760980.484039:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760980.484049:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 430 00000100:00080000:1.0:1789760980.484050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760980.484055:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789760980.484061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789760980.484068:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789760980.484074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048686464:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789760980.484077:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789760980.484096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048686976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760980.484127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048686592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760980.484183:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760980.484201:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760980/real 1789760980] req@ffff9811e1a21880 x1876700048686592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760980.484216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048686592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760980.484236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760980.484241:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760980.484246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048686592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760980.484253:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789760980.484300:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760980.484318:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760980/real 1789760980] req@ffff9811e1a20700 x1876700048686720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760980.484337:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048686720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760980.484351:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760980.484357:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760980.484362:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789760980.484372:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760980.484404:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760980.484423:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760980/real 1789760980] req@ffff9811e1a23100 x1876700048686848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760980.484450:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048686848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760980.484458:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760980.484461:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760980.484472:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789760980.484493:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789760980.484534:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789760980.484562:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760980/real 1789760980] req@ffff9811e1a23480 x1876700048686976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760980.484576:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048686976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760980.484590:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789760980.484594:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789760980.484599:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048686976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760980.739653:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760981.379724:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760981.763583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760982.403634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760982.787564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760983.427623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760983.811591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760984.451706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760984.835583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760985.475649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760985.603575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760985.603584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760985.603588:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760985.603596:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 435 00000100:00080000:2.0:1789760985.603606:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760985.603612:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760985.603643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048687104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760985.603670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760985.603673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760985.603675:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760985.603678:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 435 00000100:00080000:2.0:1789760985.603681:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760985.603685:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760985.603694:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760985.603706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048687232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760985.603719:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048687104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760985.603738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760985.603741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760985.603743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760985.603759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 435 00000100:00080000:2.0:1789760985.603764:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760985.603769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760985.603774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760985.603785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048687360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760985.603793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760985.603796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760985.603798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760985.603801:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 435 00000100:00080000:2.0:1789760985.603804:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760985.603806:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760985.603811:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789760985.603817:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789760985.603835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048687488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760985.603844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760985.603847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760985.603848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760985.603854:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 435 00000100:00080000:2.0:1789760985.603867:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760985.603872:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760985.603877:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760985.603891:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048687616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760985.603947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760985/real 1789760985] req@ffff9811d863a680 x1876700048687104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760985.603970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048687104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760985.603995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760985.603999:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760985.604004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048687104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760985.604135:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048687232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789760985.604206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760985.604222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760985/real 1789760985] req@ffff9811d8639880 x1876700048687232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760985.604236:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048687232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760985.604245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760985.604248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760985.604252:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048687232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760985.604259:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789760985.604279:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760985.604309:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760985/real 1789760985] req@ffff9811d8639500 x1876700048687360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760985.604322:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048687360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760985.604330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760985.604334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760985.604337:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760985.604351:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789760985.604375:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760985.604386:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760985/real 1789760985] req@ffff9811d8639f80 x1876700048687488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760985.604397:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048687488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760985.604404:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760985.604407:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760985.604410:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760985.604417:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789760985.604436:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760985.604447:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760985/real 1789760985] req@ffff9811d8638a80 x1876700048687616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760985.604487:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048687616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760985.604496:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760985.604499:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760985.604503:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048687616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760985.859514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760986.499577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760986.883641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760987.523664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760987.907562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760988.547627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760988.931577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760989.571654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760989.955550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760990.595595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760990.723588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760990.723599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760990.723603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760990.723613:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 440 00000100:00080000:2.0:1789760990.723625:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760990.723632:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760990.723671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048687744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760990.723722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760990.723726:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760990.723728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760990.723733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 440 00000100:00080000:2.0:1789760990.723737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760990.723753:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760990.723763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760990.723777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048687872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760990.723790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789760990.723813:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048687744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760990.723814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760990.723821:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760990.723829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 440 00000100:00080000:2.0:1789760990.723835:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760990.723842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760990.723849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760990.723869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048688000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760990.723884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760990.723888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760990.723890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760990.723896:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 440 00000800:00000100:3.0:1789760990.723903:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789760990.723905:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760990.723913:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760990.723921:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760990.723939:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048688128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760990.723949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760990.723953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760990.723955:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760990.723959:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 440 00000100:00100000:3.0:1789760990.723965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760990/real 1789760990] req@ffff9811d863b480 x1876700048687744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760990.723967:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760990.723984:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789760990.723997:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048687744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789760990.724002:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789760990.724008:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760990.724013:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760990.724018:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048687744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789760990.724026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048688256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760990.724031:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048687872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789760990.724063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760990.724075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760990/real 1789760990] req@ffff9811d8638000 x1876700048687872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760990.724085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048687872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760990.724091:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760990.724093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760990.724096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048687872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760990.724120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789760990.724151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760990.724159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760990/real 1789760990] req@ffff9811d863b800 x1876700048688000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760990.724167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048688000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760990.724173:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760990.724175:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760990.724177:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760990.724181:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789760990.724192:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760990.724198:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760990/real 1789760990] req@ffff9811d8638e00 x1876700048688128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760990.724205:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048688128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760990.724213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760990.724216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760990.724220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760990.724226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789760990.724247:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789760990.724258:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760990/real 1789760990] req@ffff9811d8638380 x1876700048688256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760990.724281:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048688256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760990.724287:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760990.724289:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760990.724291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760990.979553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760991.619621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760992.003532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760992.643721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760993.027597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760993.667739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760994.051609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789760994.691649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760995.075613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760995.715632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760995.843568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760995.843576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760995.843579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760995.843588:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 445 00000100:00080000:2.0:1789760995.843597:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760995.843603:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789760995.843635:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048688384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760995.843661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760995.843664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760995.843665:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760995.843679:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 445 00000100:00080000:2.0:1789760995.843682:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760995.843687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760995.843697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760995.843715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048688512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760995.843722:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048688384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789760995.843727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760995.843731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760995.843733:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760995.843736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 445 00000100:00080000:2.0:1789760995.843753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760995.843757:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760995.843761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760995.843772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048688640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760995.843779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760995.843782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760995.843783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760995.843786:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 445 00000100:00080000:2.0:1789760995.843789:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760995.843791:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760995.843794:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760995.843802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048688768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760995.843809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760995.843812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789760995.843813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760995.843817:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 445 00000100:00100000:3.0:1789760995.843820:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048688512:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789760995.843821:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760995.843828:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789760995.843832:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789760995.843844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048688896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760995.843853:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760995.843886:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760995.843906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760996.099541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760996.739688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760997.123814:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760997.763680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789760998.147523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789760998.787874:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789760998.915734:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789760998.915781:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789760998.915798:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760998.915848:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760998.915887:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760998.915897:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789760998.915906:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789760998.916541:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760995/real 1789760998] req@ffff9811d8638e00 x1876700048688896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760998.916584:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048688896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760998.916597:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760998.916601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760998.916606:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760998.916615:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760995/real 1789760998] req@ffff9811d8638700 x1876700048688512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760998.916631:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048688512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760998.916641:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760998.916658:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760998.916662:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048688512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760998.916670:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760995/real 1789760998] req@ffff9811d863a300 x1876700048688640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760998.916692:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048688640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760998.916702:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760998.916706:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760998.916711:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789760998.916721:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760995/real 1789760998] req@ffff9811d8638380 x1876700048688768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760998.916744:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048688768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760998.916752:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760998.916756:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760998.916759:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048688768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789760998.916767:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760995/real 1789760998] req@ffff9811d863aa00 x1876700048688384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760998.916781:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048688384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760998.916788:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760998.916791:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789760998.916795:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048688384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789760999.171552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789760999.811601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761000.195573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761000.835614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761000.963720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761000.963735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761000.963740:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761000.963754:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 450 00000100:00080000:0.0:1789761000.963768:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761000.963777:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761000.963830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048689024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761000.963873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761000.963878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761000.963880:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761000.963885:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 450 00000100:00080000:0.0:1789761000.963890:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761000.963894:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761000.963906:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761000.963932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048689152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761000.963948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761000.963954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761000.963958:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761000.963963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048689024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761000.963965:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 450 00000100:00080000:0.0:1789761000.963999:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761000.964007:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761000.964016:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761000.964033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048689280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761000.964047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761000.964055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761000.964056:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761000.964061:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761000.964070:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 450 00000100:00080000:0.0:1789761000.964077:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761000.964084:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761000.964090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761000.964106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048689408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761000.964121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761000.964122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761000/real 1789761000] req@ffff9811c2e29880 x1876700048689024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761000.964136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761000.964141:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761000.964154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 450 00000100:00080000:0.0:1789761000.964163:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761000.964170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048689024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761000.964174:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761000.964187:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789761000.964225:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761000.964231:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761000.964235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048689024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761000.964264:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048689152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761000.964293:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761000.964306:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761000/real 1789761000] req@ffff9811c2e2ad80 x1876700048689152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761000.964329:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048689536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761000.964330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048689152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761000.964531:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761000.964537:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761000.964541:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048689152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761000.964550:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761000.964579:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761000.964592:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761000/real 1789761000] req@ffff9811c2e29c00 x1876700048689280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761000.964605:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048689280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761000.964613:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761000.964617:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761000.964620:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761000.964649:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761000.964687:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761000.964698:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761000/real 1789761000] req@ffff9811c2e2a680 x1876700048689408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761000.964711:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048689408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761000.964720:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761000.964723:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761000.964726:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761000.964745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761000.964773:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761000.964785:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761000/real 1789761000] req@ffff9811c2e28700 x1876700048689536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761000.964796:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048689536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761000.964803:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761000.964807:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761000.964809:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761001.219530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761001.859575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761002.243545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761002.883667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761003.267514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761003.907660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761004.291523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761004.931707:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761005.315502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761005.955629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761006.083608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761006.083617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761006.083620:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761006.083628:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 455 00000100:00080000:0.0:1789761006.083637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761006.083643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761006.083676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048689664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761006.083706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761006.083710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761006.083712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761006.083724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 455 00000100:00080000:0.0:1789761006.083728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761006.083732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761006.083742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761006.083755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048689792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761006.083764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761006.083766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761006.083768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761006.083787:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 455 00000100:00080000:0.0:1789761006.083794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761006.083796:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048689664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761006.083800:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761006.083813:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761006.083829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048689920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761006.083839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761006.083841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761006.083842:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761006.083847:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 455 00000100:00080000:0.0:1789761006.083849:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761006.083851:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761006.083856:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761006.083865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048690048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761006.083871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761006.083873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761006.083875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761006.083880:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 455 00000100:00080000:0.0:1789761006.083882:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761006.083885:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761006.083888:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761006.083895:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761006.083900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048690176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761006.083950:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761006/real 1789761006] req@ffff9811c2e2bb80 x1876700048689664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761006.083969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048689664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761006.083979:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761006.083982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761006.083985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048689664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761006.083992:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048689792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761006.084009:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761006.084017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761006/real 1789761006] req@ffff9811c2e2b100 x1876700048689792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761006.084034:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048689792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761006.084041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761006.084043:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761006.084046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048689792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761006.084070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761006.084083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761006.084090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761006/real 1789761006] req@ffff9811c2e2aa00 x1876700048689920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761006.084121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048689920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761006.084127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761006.084129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761006.084132:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048689920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761006.084136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761006.084153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761006.084160:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761006/real 1789761006] req@ffff9811c2e29180 x1876700048690048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761006.084168:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048690048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761006.084173:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761006.084175:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761006.084176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761006.084180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761006.084191:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761006.084200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761006/real 1789761006] req@ffff9811c2e28e00 x1876700048690176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761006.084211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048690176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761006.084242:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761006.084246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761006.084250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761006.339502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761006.979682:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761007.363508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761008.003650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761008.387629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761009.027674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761009.411535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761010.051640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761010.435556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761011.075615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761011.203565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761011.203577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761011.203581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761011.203592:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 460 00000100:00080000:3.0:1789761011.203606:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761011.203613:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761011.203658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048690304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761011.203687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761011.203690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761011.203692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761011.203695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 460 00000100:00080000:3.0:1789761011.203697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761011.203699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761011.203709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761011.203717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048690432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761011.203745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761011.203750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789761011.203753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048690304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761011.203753:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761011.203761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 460 00000100:00080000:3.0:1789761011.203764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761011.203767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761011.203771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761011.203781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048690560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761011.203788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761011.203791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761011.203793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761011.203795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 460 00000100:00080000:3.0:1789761011.203797:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761011.203799:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761011.203803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761011.203809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048690688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761011.203816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761011.203819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761011.203820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761011.203822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 460 00000800:00000100:1.0:1789761011.203830:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761011.203831:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761011.203837:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761011.203842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761011.203857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048690816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761011.203894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761011/real 1789761011] req@ffff9811e1a20000 x1876700048690304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761011.203925:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048690304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761011.203934:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761011.203937:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761011.203941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048690304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761011.203963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048690432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761011.203986:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761011.204001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761011/real 1789761011] req@ffff9811e1a23800 x1876700048690432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761011.204015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048690432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761011.204025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761011.204029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761011.204032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048690432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761011.204037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761011.204060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761011.204072:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761011/real 1789761011] req@ffff9811e1a21f80 x1876700048690560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761011.204086:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048690560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761011.204106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761011.204109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761011.204111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761011.204116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761011.204133:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761011.204141:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761011/real 1789761011] req@ffff9811e1a20a80 x1876700048690688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761011.204149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048690688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761011.204154:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761011.204156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761011.204158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761011.204162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761011.204173:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761011.204178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761011/real 1789761011] req@ffff9811e1a20e00 x1876700048690816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761011.204185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048690816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761011.204190:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761011.204192:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761011.204199:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048690816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761011.459531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761012.099622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761012.483568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761013.123637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761013.507525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761014.147588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761014.531582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761015.171637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761015.555542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761016.195642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761016.323615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761016.323630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761016.323636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761016.323648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 465 00000100:00080000:0.0:1789761016.323662:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761016.323671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761016.323719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048690944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761016.323754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761016.323758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761016.323760:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761016.323765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 465 00000100:00080000:0.0:1789761016.323768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761016.323772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761016.323783:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761016.323799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048691072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761016.323814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761016.323815:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048690944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761016.323821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761016.323825:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761016.323830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 465 00000100:00080000:0.0:1789761016.323850:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761016.323856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761016.323861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761016.323874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048691200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761016.323886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761016.323890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1789761016.323893:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761016.323894:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761016.323901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 465 00000100:00080000:0.0:1789761016.323906:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761016.323910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761016.323916:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761016.323928:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048691328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761016.323938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761016.323942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761016.323945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761016.323950:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 465 00000100:00100000:1.0:1789761016.323952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761016/real 1789761016] req@ffff9811c2e28380 x1876700048690944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761016.323954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761016.323963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761016.323971:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761016.323981:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048690944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761016.323986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048691456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761016.323999:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761016.324005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761016.324013:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048690944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761016.324048:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048691072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761016.324068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761016.324077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761016/real 1789761016] req@ffff9811c2e28a80 x1876700048691072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761016.324092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048691072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761016.324098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761016.324101:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761016.324103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048691072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761016.324108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761016.324131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761016.324139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761016/real 1789761016] req@ffff9811c2e29f80 x1876700048691200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761016.324147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048691200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761016.324152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761016.324154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761016.324156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761016.324160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761016.324172:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761016.324178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761016/real 1789761016] req@ffff9811c2e29500 x1876700048691328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761016.324186:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048691328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761016.324191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761016.324192:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761016.324194:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761016.324198:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761016.324208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761016.324213:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761016/real 1789761016] req@ffff9811c2e2b480 x1876700048691456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761016.324232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048691456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761016.324238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761016.324240:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761016.324242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761016.579573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761017.219604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761017.603615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761018.243666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761018.627618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761019.267654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761019.651580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761020.291635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761020.675560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761021.315664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761021.443575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761021.443586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761021.443591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761021.443602:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 471 00000100:00080000:0.0:1789761021.443622:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761021.443634:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761021.443667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048691584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761021.443694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761021.443703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761021.443706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761021.443711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 471 00000100:00080000:0.0:1789761021.443715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761021.443721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761021.443730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761021.443731:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048691584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761021.443748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048691712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761021.443766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761021.443770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761021.443773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761021.443779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 471 00000800:00000100:3.0:1789761021.443794:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761021.443798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761021.443806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761021.443815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761021.443828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048691840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761021.443840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761021.443845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1789761021.443847:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761021/real 1789761021] req@ffff9811c2e2b800 x1876700048691584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761021.443848:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761021.443856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 471 00000100:00080000:0.0:1789761021.443862:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761021.443866:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761021.443871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761021.443882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048691968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761021.443891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761021.443895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761021.443897:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761021.443901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 471 00000100:00080000:0.0:1789761021.443905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761021.443908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761021.443912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761021.443921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048692096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761021.443942:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048691584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761021.443963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761021.443968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761021.443982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048691584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761021.444124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048691712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761021.444159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761021.444179:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761021/real 1789761021] req@ffff9811c2e28000 x1876700048691712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761021.444195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048691712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761021.444210:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761021.444216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761021.444221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048691712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761021.444230:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761021.444258:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761021.444270:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761021/real 1789761021] req@ffff9811c2e2a300 x1876700048691840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761021.444280:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048691840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761021.444289:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761021.444292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761021.444296:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761021.444302:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761021.444324:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761021.444346:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761021/real 1789761021] req@ffff9811c2e2b480 x1876700048691968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761021.444354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048691968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761021.444360:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761021.444362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761021.444364:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048691968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761021.444368:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761021.444382:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761021.444388:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761021/real 1789761021] req@ffff9811c2e29500 x1876700048692096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761021.444399:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048692096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761021.444407:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761021.444410:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761021.444413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761021.699604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761022.339597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761022.723619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761023.363593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761023.747586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761024.387582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761024.771575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761025.411621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761025.795579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761026.435600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761026.563580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761026.563591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761026.563596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761026.563607:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 476 00000100:00080000:2.0:1789761026.563620:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761026.563630:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761026.563667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048692224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761026.563694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761026.563698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761026.563701:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761026.563706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 476 00000100:00080000:2.0:1789761026.563710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761026.563715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761026.563725:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761026.563726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048692224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761026.563742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048692352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761026.563756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761026.563760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761026.563762:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761026.563766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 476 00000100:00080000:2.0:1789761026.563778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761026.563783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789761026.563794:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761026.563804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761026.563818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048692480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761026.563830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761026.563834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761026.563836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761026.563840:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 476 00000100:00080000:2.0:1789761026.563845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761026.563851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761026.563851:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761026/real 1789761026] req@ffff9811d863b800 x1876700048692224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761026.563860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761026.563873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048692608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761026.563885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761026.563886:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048692224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761026.563891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761026.563895:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761026.563905:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 476 00000100:00080000:2.0:1789761026.563911:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761026.563912:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761026.563917:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789761026.563918:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:2.0:1789761026.563925:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761026.563925:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048692224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761026.563942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048692736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761026.563951:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048692352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761026.563981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761026.563996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761026/real 1789761026] req@ffff9811d8638000 x1876700048692352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761026.564010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048692352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761026.564019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761026.564033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761026.564037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048692352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761026.564045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761026.564065:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761026.564074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761026/real 1789761026] req@ffff9811d863b480 x1876700048692480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761026.564094:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048692480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761026.564100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761026.564102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761026.564104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761026.564108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761026.564123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761026.564130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761026/real 1789761026] req@ffff9811d8638a80 x1876700048692608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761026.564138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048692608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761026.564143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761026.564144:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761026.564146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761026.564158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761026.564169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761026.564174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761026/real 1789761026] req@ffff9811d8639f80 x1876700048692736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761026.564182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048692736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761026.564195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761026.564198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761026.564200:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048692736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761026.819571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761027.459601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761027.843567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761028.483683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761028.867520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761029.507616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761029.891615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761030.531625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761030.915554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761031.555680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761031.683619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761031.683630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761031.683634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761031.683644:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 481 00000100:00080000:0.0:1789761031.683657:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761031.683664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761031.683717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048692864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761031.683750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761031.683753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761031.683755:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761031.683758:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 481 00000100:00080000:0.0:1789761031.683761:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761031.683764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761031.683774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761031.683784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048692992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761031.683794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761031.683796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761031.683797:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761031.683800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 481 00000100:00080000:0.0:1789761031.683821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761031.683825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761031.683829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761031.683838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048693120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761031.683847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761031.683852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761031.683854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761031.683856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048692864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761031.683860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 481 00000100:00080000:0.0:1789761031.683868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761031.683876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761031.683879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761031.683888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048693248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761031.683893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761031.683897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761031.683899:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761031.683902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 481 00000100:00080000:0.0:1789761031.683904:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761031.683907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761031.683909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761031.683916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048693376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761031.683952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761031.684001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761031/real 1789761031] req@ffff9811c2e29f80 x1876700048692864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761031.684022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048692864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761031.684035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761031.684040:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761031.684044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048692864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761031.684053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048692992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761031.684079:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761031.684094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761031/real 1789761031] req@ffff9811c2e28a80 x1876700048692992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761031.684121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048692992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761031.684132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761031.684135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761031.684138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048692992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761031.684165:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761031.684188:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761031.684209:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761031/real 1789761031] req@ffff9811c2e28380 x1876700048693120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761031.684222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048693120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761031.684231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761031.684235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761031.684239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761031.684246:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761031.684269:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761031.684282:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761031/real 1789761031] req@ffff9811c2e28e00 x1876700048693248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761031.684295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048693248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761031.684320:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761031.684324:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761031.684328:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761031.684335:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761031.684358:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761031.684369:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761031/real 1789761031] req@ffff9811c2e29180 x1876700048693376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761031.684381:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048693376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761031.684389:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761031.684392:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761031.684396:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761031.939563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761032.579650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761032.963524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761033.603668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761033.987574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761034.627650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761035.011515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761035.651734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761036.035592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761036.675720:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761036.803606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761036.803618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761036.803622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761036.803634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 486 00000100:00080000:0.0:1789761036.803647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761036.803654:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761036.803709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048693504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761036.803738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761036.803741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761036.803742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761036.803746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 486 00000100:00080000:0.0:1789761036.803749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761036.803752:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761036.803760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761036.803771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048693632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761036.803779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761036.803781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761036.803782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761036.803803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 486 00000100:00080000:0.0:1789761036.803806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761036.803810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761036.803814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761036.803824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048693504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761036.803825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048693760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761036.803839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761036.803842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761036.803843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761036.803846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 486 00000100:00080000:0.0:1789761036.803849:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761036.803851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761036.803855:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761036.803862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048693888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761036.803868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761036.803870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761036.803872:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761036.803875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 486 00000100:00080000:0.0:1789761036.803878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761036.803881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761036.803884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761036.803892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048694016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761036.803905:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761036.803966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761036/real 1789761036] req@ffff9811c2e2aa00 x1876700048693504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761036.804009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048693504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761036.804020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761036.804023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761036.804026:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048693504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761036.804032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048693632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761036.804053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761036.804063:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761036/real 1789761036] req@ffff9811c2e2b100 x1876700048693632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761036.804072:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048693632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761036.804077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761036.804080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761036.804082:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048693632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761036.804105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761036.804115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761036.804122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761036/real 1789761036] req@ffff9811c2e2bb80 x1876700048693760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761036.804129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048693760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761036.804144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761036.804147:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761036.804149:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761036.804153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761036.804167:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761036.804173:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761036/real 1789761036] req@ffff9811c2e28700 x1876700048693888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761036.804181:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048693888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761036.804186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761036.804188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761036.804190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048693888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761036.804193:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761036.804203:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761036.804209:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761036/real 1789761036] req@ffff9811c2e2a680 x1876700048694016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761036.804216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048694016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761036.804221:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761036.804223:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761036.804231:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761037.059605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761037.699614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761038.083543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761038.723634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761039.107600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761039.747654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761040.131559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761040.771664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761041.155599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761041.795659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761041.923605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761041.923619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761041.923645:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761041.923658:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 491 00000100:00080000:2.0:1789761041.923674:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761041.923683:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761041.923736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048694144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761041.923769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761041.923774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761041.923776:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761041.923780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 491 00000100:00080000:2.0:1789761041.923785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761041.923791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761041.923803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761041.923818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048694272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761041.923835:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048694144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761041.923837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761041.923856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761041.923861:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761041.923868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 491 00000100:00080000:2.0:1789761041.923873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761041.923879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761041.923885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761041.923901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048694400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761041.923925:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761041.923929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761041.923935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761041.923938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761041.923945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 491 00000100:00080000:2.0:1789761041.923952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761041.923957:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761041.923965:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761041.923979:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761041/real 1789761041] req@ffff9811d8639500 x1876700048694144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761041.923984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048694528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761041.924018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761041.924018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048694144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761041.924040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761041.924044:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761041.924045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761041.924065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789761041.924066:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 491 00000100:00100000:3.0:1789761041.924074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048694144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761041.924075:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761041.924091:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761041.924100:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761041.924107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048694272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761041.924119:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048694656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761041.924131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761041.924147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761041/real 1789761041] req@ffff9811d8639880 x1876700048694272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761041.924156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048694272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761041.924163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761041.924166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761041.924181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048694272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761041.924187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761041.924211:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761041.924219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761041/real 1789761041] req@ffff9811d863a680 x1876700048694400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761041.924226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048694400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761041.924231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761041.924233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761041.924236:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761041.924239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761041.924250:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761041.924256:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761041/real 1789761041] req@ffff9811d863ad80 x1876700048694528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761041.924263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048694528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761041.924267:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761041.924269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761041.924271:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761041.924282:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761041.924292:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761041.924307:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761041/real 1789761041] req@ffff9811d8639c00 x1876700048694656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761041.924315:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048694656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761041.924320:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761041.924322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761041.924341:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048694656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761042.179567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761042.819686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761043.203555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761043.843635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761044.227539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761044.867605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761045.251560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761045.891625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761046.275538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761046.915702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761047.043648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761047.043664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761047.043673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761047.043713:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 496 00000100:00080000:2.0:1789761047.043756:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761047.043770:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761047.043855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048694784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761047.043901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761047.043908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761047.043912:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761047.043920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 496 00000100:00080000:2.0:1789761047.043928:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761047.043937:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761047.043954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761047.043998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048694912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761047.044027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048694784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761047.044033:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761047.044045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761047.044049:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761047.044055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 496 00000100:00080000:2.0:1789761047.044060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761047.044065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761047.044072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761047.044087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048695040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761047.044116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761047.044131:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761047.044134:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761047.044140:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 496 00000100:00080000:2.0:1789761047.044146:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761047.044152:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761047.044159:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761047.044163:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761047.044179:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048695168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761047.044194:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761047.044199:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761047.044202:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761047.044207:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 496 00000100:00080000:2.0:1789761047.044210:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761047.044214:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761047.044222:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761047.044232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048695296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761047.044254:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761047/real 1789761047] req@ffff9811d863b100 x1876700048694784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761047.044303:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048694784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761047.044317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761047.044323:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761047.044328:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048694784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761047.044373:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048694912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761047.044404:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761047.044435:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761047/real 1789761047] req@ffff9811d863bb80 x1876700048694912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761047.044449:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048694912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761047.044459:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761047.044499:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761047.044504:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048694912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761047.044512:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761047.044536:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761047.044549:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761047/real 1789761047] req@ffff9811d8639180 x1876700048695040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761047.044562:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048695040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761047.044571:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761047.044575:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761047.044579:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761047.044586:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761047.044608:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761047.044619:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761047/real 1789761047] req@ffff9811d8639c00 x1876700048695168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761047.044630:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048695168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761047.044646:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761047.044650:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761047.044654:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761047.044661:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761047.044683:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761047.044694:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761047/real 1789761047] req@ffff9811d863ad80 x1876700048695296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761047.044706:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048695296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761047.044716:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761047.044719:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761047.044723:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761047.299561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761047.939637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761048.323532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761048.963687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761049.347546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761049.987633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761050.371522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761051.011638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761051.395552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761052.035605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761052.163578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761052.163586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761052.163589:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761052.163599:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 501 00000100:00080000:0.0:1789761052.163608:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761052.163614:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761052.163646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048695424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761052.163665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761052.163668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761052.163669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761052.163672:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 501 00000100:00080000:0.0:1789761052.163674:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761052.163676:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761052.163683:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761052.163691:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048695552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761052.163700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761052.163702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761052.163716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761052.163719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 501 00000100:00080000:0.0:1789761052.163723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761052.163726:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761052.163731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761052.163739:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048695424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761052.163741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048695680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761052.163755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761052.163757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761052.163758:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761052.163760:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 501 00000100:00080000:0.0:1789761052.163762:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761052.163765:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761052.163768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761052.163778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048695808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761052.163784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761052.163786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761052.163787:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761052.163790:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 501 00000100:00080000:0.0:1789761052.163793:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761052.163796:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761052.163799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761052.163813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048695936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761052.163813:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761052.163857:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761052/real 1789761052] req@ffff9811c2e29c00 x1876700048695424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761052.163891:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048695424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761052.163902:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761052.163907:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761052.163911:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048695424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761052.163919:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048695552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761052.163941:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761052.163952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761052/real 1789761052] req@ffff9811c2e2ad80 x1876700048695552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761052.163965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048695552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761052.163974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761052.163977:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761052.163981:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048695552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761052.164001:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761052.164017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761052.164026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761052/real 1789761052] req@ffff9811c2e29880 x1876700048695680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761052.164038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048695680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761052.164059:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761052.164062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761052.164066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761052.164072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761052.164090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761052.164100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761052/real 1789761052] req@ffff9811c2e2a680 x1876700048695808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761052.164112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048695808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761052.164121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761052.164124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761052.164127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761052.164133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761052.164147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761052.164155:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761052/real 1789761052] req@ffff9811c2e28700 x1876700048695936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761052.164167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048695936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761052.164175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761052.164178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761052.164188:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048695936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761052.419570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761053.059623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761053.443575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761054.083612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761054.467553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761055.107689:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761055.491609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761056.131629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761056.515578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761057.155658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761057.283586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761057.283608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761057.283613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761057.283623:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 506 00000100:00080000:2.0:1789761057.283636:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761057.283643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761057.283687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048696064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761057.283734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761057.283737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761057.283739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761057.283742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 506 00000100:00080000:2.0:1789761057.283745:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761057.283749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761057.283758:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761057.283770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048696192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761057.283780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761057.283784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761057.283785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048696064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761057.283786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761057.283793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 506 00000100:00080000:2.0:1789761057.283798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761057.283801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761057.283804:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761057.283812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048696320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761057.283819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761057.283822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761057.283823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761057.283826:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 506 00000100:00080000:2.0:1789761057.283835:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761057.283838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761057.283843:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761057.283852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048696448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761057.283863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761057.283866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761057.283868:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761057.283872:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 506 00000800:00000100:3.0:1789761057.283877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761057.283879:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761057.283893:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761057.283900:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761057.283914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048696576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761057.283929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761057/real 1789761057] req@ffff9811d863a680 x1876700048696064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761057.283945:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048696064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761057.283954:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761057.283958:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761057.283962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048696064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761057.283991:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048696192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761057.284014:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761057.284025:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761057/real 1789761057] req@ffff9811d8639880 x1876700048696192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761057.284034:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048696192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761057.284041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761057.284043:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761057.284045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048696192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761057.284049:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761057.284065:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761057.284083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761057/real 1789761057] req@ffff9811d8639500 x1876700048696320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761057.284093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048696320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761057.284103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761057.284107:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761057.284112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761057.284120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761057.284150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761057.284164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761057/real 1789761057] req@ffff9811d8639f80 x1876700048696448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761057.284178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048696448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761057.284265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761057.284268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761057.284271:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761057.284278:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761057.284297:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761057.284309:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761057/real 1789761057] req@ffff9811d8638a80 x1876700048696576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761057.284320:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048696576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761057.284442:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761057.284446:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761057.284450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761057.539536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761058.179593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761058.563568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761059.203700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761059.587580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761060.227639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761060.611597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761061.251626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761061.635565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761062.275602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761062.403595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761062.403606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761062.403610:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761062.403622:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 511 00000100:00080000:2.0:1789761062.403641:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761062.403649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761062.403683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048696704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761062.403708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761062.403731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761062.403734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761062.403741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 511 00000100:00080000:2.0:1789761062.403746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761062.403752:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761062.403763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761062.403768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048696704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761062.403784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048696832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761062.403798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761062.403801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761062.403803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761062.403806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 511 00000100:00080000:2.0:1789761062.403818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761062.403822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761062.403826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761062.403836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048696960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761062.403843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761062.403845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761062.403847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761062.403850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 511 00000100:00080000:2.0:1789761062.403852:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761062.403855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761062.403866:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761062.403868:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048696832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761062.403876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048697088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761062.403886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761062.403889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761062.403890:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761062.403894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 511 00000100:00100000:3.0:1789761062.403897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696960:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789761062.403899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761062.403906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761062.403911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761062.403923:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048697216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761062.403931:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761062.403953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761062.659569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761063.299581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761063.683574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761064.323588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761064.707534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761065.347610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761065.475743:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789761065.475788:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761065.475806:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761065.475843:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761065.475853:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761065.475865:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761065.475878:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789761065.475893:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761062/real 1789761065] req@ffff9811d863aa00 x1876700048697088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761065.475922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048697088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761065.475951:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761065.475957:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761065.475961:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761065.475970:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761062/real 1789761065] req@ffff9811d8638000 x1876700048696832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761065.475986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048696832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761065.476017:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761065.476021:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761065.476025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048696832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761065.476033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761062/real 1789761065] req@ffff9811d863b800 x1876700048696960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761065.476047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048696960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761065.476056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761065.476060:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761065.476064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048696960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761065.476071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761062/real 1789761065] req@ffff9811d863b480 x1876700048696704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761065.476085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048696704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761065.476093:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761065.476097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761065.476100:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048696704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761065.476128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761062/real 1789761065] req@ffff9811d8638380 x1876700048697216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761065.476142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048697216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761065.476175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761065.476180:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761065.476184:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761065.731568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761066.371599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761066.755555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761067.395595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761067.523648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761067.523663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761067.523669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761067.523682:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 517 00000100:00080000:0.0:1789761067.523696:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761067.523705:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761067.523755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048697344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761067.523806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761067.523815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761067.523817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761067.523821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 517 00000100:00080000:0.0:1789761067.523825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761067.523828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761067.523839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761067.523852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048697472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761067.523863:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048697344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761067.523864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761067.523872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761067.523875:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761067.523879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 517 00000100:00080000:0.0:1789761067.523881:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761067.523884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761067.523887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761067.523897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048697600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761067.523907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761067.523911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761067.523913:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761067.523917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 517 00000100:00080000:0.0:1789761067.523929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761067.523934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761067.523941:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761067.523957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048697728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761067.523966:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761067.523977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761067.523982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761067.523985:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761067.523990:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 517 00000100:00080000:0.0:1789761067.524005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761067.524011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761067.524018:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761067.524032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761067/real 1789761067] req@ffff9811c2e2bb80 x1876700048697344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761067.524036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048697856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761067.524080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048697344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761067.524121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761067.524127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761067.524146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048697344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761067.524180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048697472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761067.524210:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761067.524227:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761067/real 1789761067] req@ffff9811c2e2b100 x1876700048697472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761067.524244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048697472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761067.524253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761067.524257:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761067.524262:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048697472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761067.524277:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761067.524301:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761067.524313:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761067/real 1789761067] req@ffff9811c2e2aa00 x1876700048697600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761067.524326:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048697600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761067.524334:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761067.524337:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761067.524341:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761067.524347:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761067.524507:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761067.524526:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761067/real 1789761067] req@ffff9811c2e29180 x1876700048697728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761067.524539:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048697728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761067.524547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761067.524551:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761067.524554:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761067.524561:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761067.524583:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761067.524594:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761067/real 1789761067] req@ffff9811c2e28e00 x1876700048697856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761067.524615:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048697856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761067.524624:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761067.524629:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761067.524633:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048697856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761067.779525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761068.419614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761068.803523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761069.443641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761069.827514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761070.467610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761070.851517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761071.491636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761071.875532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761072.515680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761072.643580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761072.643589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761072.643592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761072.643601:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 522 00000100:00080000:2.0:1789761072.643614:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761072.643636:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761072.643663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048697984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761072.643683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761072.643686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761072.643687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761072.643690:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 522 00000100:00080000:2.0:1789761072.643692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761072.643694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761072.643702:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761072.643710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048698112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761072.643719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761072.643723:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048697984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761072.643724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761072.643727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761072.643735:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 522 00000100:00080000:2.0:1789761072.643737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761072.643739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761072.643742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761072.643752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048698240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761072.643758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761072.643760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761072.643767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761072.643770:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 522 00000100:00080000:2.0:1789761072.643781:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761072.643784:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761072.643788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761072.643794:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761072.643799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048698368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761072.643808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761072.643810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761072.643811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761072.643814:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 522 00000100:00080000:2.0:1789761072.643816:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761072.643818:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761072.643821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761072.643830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048698496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761072.643837:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761072/real 1789761072] req@ffff9811d863a300 x1876700048697984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761072.643855:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048697984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761072.643867:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761072.643871:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761072.643875:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048697984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761072.643911:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048698112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761072.643930:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761072.643940:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761072/real 1789761072] req@ffff9811d8638700 x1876700048698112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761072.643948:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048698112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761072.643964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761072.643967:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761072.643969:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048698112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761072.643973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761072.643987:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761072.643994:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761072/real 1789761072] req@ffff9811d8638e00 x1876700048698240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761072.644001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048698240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761072.644006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761072.644008:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761072.644010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761072.644014:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761072.644024:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761072.644030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761072/real 1789761072] req@ffff9811d8638380 x1876700048698368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761072.644042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048698368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761072.644050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761072.644053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761072.644063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761072.644070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761072.644091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761072.644120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761072/real 1789761072] req@ffff9811d863b480 x1876700048698496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761072.644141:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048698496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761072.644150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761072.644152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761072.644155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761072.899513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761073.539673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761073.923521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761074.563695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761074.947540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761075.587658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761075.971587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761076.611673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761076.995631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761077.635585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761077.763599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761077.763611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761077.763615:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761077.763626:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 527 00000100:00080000:2.0:1789761077.763639:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761077.763646:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761077.763698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048698624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761077.763759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761077.763764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761077.763766:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761077.763771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 527 00000100:00080000:2.0:1789761077.763775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761077.763779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761077.763790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761077.763809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048698624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761077.763816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048698752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761077.763832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761077.763837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761077.763839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761077.763843:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 527 00000100:00080000:2.0:1789761077.763846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761077.763850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761077.763854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761077.763865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048698880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761077.763874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761077.763876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761077.763878:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761077.763881:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 527 00000800:00000100:3.0:1789761077.763886:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761077.763892:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761077.763898:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761077.763903:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761077.763914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048699008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761077.763921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761077.763925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761077.763927:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761077.763932:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 527 00000100:00100000:3.0:1789761077.763937:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761077/real 1789761077] req@ffff9811d863b800 x1876700048698624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761077.763939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761077.763949:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761077.763956:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761077.763960:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048698624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761077.763971:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761077.763983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048699136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761077.763995:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761077.764000:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048698624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761077.764024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048698752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761077.764043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761077.764077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761077/real 1789761077] req@ffff9811d8638000 x1876700048698752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761077.764088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048698752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761077.764094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761077.764106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761077.764109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048698752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761077.764113:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761077.764128:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761077.764136:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761077/real 1789761077] req@ffff9811d863aa00 x1876700048698880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761077.764144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048698880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761077.764149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761077.764151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761077.764153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048698880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761077.764156:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761077.764167:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761077.764173:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761077/real 1789761077] req@ffff9811d8638a80 x1876700048699008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761077.764180:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048699008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761077.764191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761077.764193:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761077.764196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761077.764200:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761077.764213:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761077.764219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761077/real 1789761077] req@ffff9811d8639f80 x1876700048699136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761077.764226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048699136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761077.764232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761077.764233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761077.764235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761078.019579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761078.659596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761079.043601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761079.683609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761080.067578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761080.707624:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761081.091578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761081.731579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761082.115604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761082.755572:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761082.883522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761082.883531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761082.883534:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761082.883543:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 532 00000100:00080000:2.0:1789761082.883552:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761082.883559:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761082.883588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048699264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761082.883611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761082.883628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761082.883630:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761082.883634:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 532 00000100:00080000:2.0:1789761082.883637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761082.883640:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761082.883649:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761082.883659:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048699392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761082.883669:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048699264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761082.883675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761082.883679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761082.883681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761082.883684:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 532 00000100:00080000:2.0:1789761082.883686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761082.883691:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761082.883694:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761082.883704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048699520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761082.883710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761082.883713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761082.883715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761082.883718:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 532 00000100:00080000:2.0:1789761082.883738:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761082.883742:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761082.883752:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761082.883754:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761082.883774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048699648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761082.883782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761082.883785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761082.883788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761082.883792:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 532 00000100:00080000:2.0:1789761082.883797:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761082.883800:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761082.883804:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761082.883811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048699776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761082.883829:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761082/real 1789761082] req@ffff9811d8639500 x1876700048699264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761082.883846:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048699264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761082.883873:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761082.883877:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761082.883880:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048699264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761082.883905:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048699392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761082.883926:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761082.883936:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761082/real 1789761082] req@ffff9811d8639880 x1876700048699392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761082.883944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048699392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761082.883950:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761082.883953:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761082.883955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048699392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761082.883959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761082.883970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761082.883976:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761082/real 1789761082] req@ffff9811d863a680 x1876700048699520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761082.883984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048699520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761082.883989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761082.883991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761082.883993:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761082.884005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761082.884037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761082.884046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761082/real 1789761082] req@ffff9811d863ad80 x1876700048699648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761082.884055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048699648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761082.884063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761082.884066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761082.884070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761082.884077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761082.884098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761082.884110:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761082/real 1789761082] req@ffff9811d8639c00 x1876700048699776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761082.884121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048699776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761082.884126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761082.884129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761082.884131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048699776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761083.139633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761083.779617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761084.163604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761084.803603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761085.187550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761085.827650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761086.211603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761086.851646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761087.235542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761087.875654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761088.003551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761088.003562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761088.003567:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761088.003580:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 537 00000100:00080000:3.0:1789761088.003593:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761088.003601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761088.003659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048699904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761088.003687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761088.003691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761088.003692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761088.003696:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 537 00000100:00080000:3.0:1789761088.003699:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761088.003704:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761088.003716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761088.003728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048699904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761088.003735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048700032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761088.003750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761088.003754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761088.003757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761088.003761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 537 00000100:00080000:3.0:1789761088.003766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761088.003770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761088.003791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761088.003808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761088.003808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048700160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761088.003820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761088.003824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761088.003826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761088.003829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 537 00000100:00080000:3.0:1789761088.003833:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761088.003836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761088.003840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761088.003848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048700288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761088.003854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761088.003857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761088.003859:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761088.003864:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761088/real 1789761088] req@ffff9811e1a21500 x1876700048699904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761088.003864:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 537 00000100:00080000:3.0:1789761088.003868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761088.003873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761088.003879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761088.003891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048700416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761088.003899:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048699904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761088.003911:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761088.003916:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761088.003920:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048699904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761088.003943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048700032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761088.003962:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761088.003972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761088/real 1789761088] req@ffff9811e1a20380 x1876700048700032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761088.004005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048700032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761088.004013:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761088.004017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761088.004021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048700032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761088.004028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761088.004051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761088.004063:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761088/real 1789761088] req@ffff9811e1a20e00 x1876700048700160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761088.004074:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048700160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761088.004080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761088.004082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761088.004091:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761088.004098:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761088.004114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761088.004122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761088/real 1789761088] req@ffff9811e1a20a80 x1876700048700288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761088.004129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048700288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761088.004144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761088.004147:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761088.004149:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761088.004153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761088.004167:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761088.004174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761088/real 1789761088] req@ffff9811e1a21f80 x1876700048700416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761088.004181:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048700416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761088.004186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761088.004188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761088.004190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761088.259556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761088.899607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761089.283533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761089.923629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761090.307559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761090.947627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761091.332510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761091.971615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761092.355555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761092.995634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761093.123617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761093.123623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761093.123627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761093.123635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 542 00000100:00080000:0.0:1789761093.123645:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761093.123651:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761093.123681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048700544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761093.123704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761093.123708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761093.123709:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761093.123715:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 542 00000100:00080000:0.0:1789761093.123720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761093.123723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761093.123732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761093.123753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048700544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761093.123754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048700672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761093.123771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761093.123774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761093.123775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761093.123779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 542 00000100:00080000:0.0:1789761093.123782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761093.123785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761093.123790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761093.123799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048700800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761093.123807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761093.123809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761093.123811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761093.123814:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 542 00000100:00080000:0.0:1789761093.123821:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761093.123825:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761093.123830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761093.123835:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761093.123841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048700928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761093.123848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761093.123851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761093.123854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761093.123862:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 542 00000100:00080000:0.0:1789761093.123866:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761093.123869:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761093.123874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761093.123884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048701056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761093.123890:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761093/real 1789761093] req@ffff9811c2e28380 x1876700048700544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761093.124123:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048700544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761093.124134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761093.124138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761093.124142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048700544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761093.124228:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048700672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761093.124252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761093.124263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761093/real 1789761093] req@ffff9811c2e28a80 x1876700048700672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761093.124272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048700672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761093.124278:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761093.124281:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761093.124283:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048700672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761093.124287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761093.124308:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761093.124316:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761093/real 1789761093] req@ffff9811c2e29f80 x1876700048700800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761093.124324:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048700800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761093.124330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761093.124332:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761093.124334:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761093.124338:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761093.124360:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761093.124365:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761093/real 1789761093] req@ffff9811c2e29500 x1876700048700928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761093.124372:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048700928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761093.124378:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761093.124379:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761093.124381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048700928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761093.124384:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761093.124393:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761093.124399:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761093/real 1789761093] req@ffff9811c2e2b480 x1876700048701056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761093.124416:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048701056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761093.124421:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761093.124424:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761093.124428:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761093.379522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761094.019638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761094.403541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761095.043651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761095.427562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761096.067640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761096.451539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761097.091596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761097.475574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761098.115639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761098.243578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761098.243587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761098.243591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761098.243600:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 547 00000100:00080000:0.0:1789761098.243609:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761098.243615:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761098.243648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048701184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761098.243672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761098.243676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761098.243677:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761098.243680:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 547 00000100:00080000:0.0:1789761098.243682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761098.243685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761098.243716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761098.243720:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048701184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761098.243730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048701312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761098.243741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761098.243744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761098.243746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761098.243749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 547 00000100:00080000:0.0:1789761098.243752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761098.243756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761098.243760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761098.243768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048701440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761098.243776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761098.243779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761098.243782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761098.243783:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761098.243788:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 547 00000100:00080000:0.0:1789761098.243792:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761098.243795:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761098.243799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761098.243808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048701568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761098.243816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761098.243818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789761098.243826:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761098/real 1789761098] req@ffff9811c2e2a300 x1876700048701184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761098.243827:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761098.243836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 547 00000100:00080000:0.0:1789761098.243842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761098.243847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761098.243852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761098.243855:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048701184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761098.243864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048701696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761098.243867:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761098.243878:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761098.243884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048701184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761098.243926:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048701312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761098.243948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761098.243959:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761098/real 1789761098] req@ffff9811c2e28000 x1876700048701312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761098.243974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048701312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761098.243993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761098.243998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761098.244002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048701312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761098.244010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761098.244043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761098.244053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761098/real 1789761098] req@ffff9811c2e2b800 x1876700048701440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761098.244062:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048701440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761098.244068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761098.244070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761098.244073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761098.244077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761098.244091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761098.244098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761098/real 1789761098] req@ffff9811c2e2b480 x1876700048701568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761098.244106:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048701568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761098.244111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761098.244113:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761098.244122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761098.244126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761098.244141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761098.244147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761098/real 1789761098] req@ffff9811c2e29500 x1876700048701696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761098.244156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048701696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761098.244161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761098.244163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761098.244165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048701696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761098.499573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761099.139703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761099.523528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761100.163607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761100.547604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761101.187633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761101.571556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761102.211631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761102.595569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761103.235583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761103.363596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761103.363607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761103.363611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761103.363623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 552 00000100:00080000:0.0:1789761103.363637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761103.363646:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761103.363731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048701824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761103.363761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761103.363765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761103.363767:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761103.363771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 552 00000100:00080000:0.0:1789761103.363774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761103.363812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761103.363821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761103.363832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048701952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761103.363840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761103.363843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761103.363862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761103.363884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 552 00000100:00080000:0.0:1789761103.363891:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761103.363898:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761103.363905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761103.363938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048702080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761103.363939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048701824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761103.363959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761103.363962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761103.363964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761103.363967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 552 00000100:00080000:0.0:1789761103.363971:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761103.363974:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761103.363979:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761103.363989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048702208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761103.363996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761103.364004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761103.364006:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761103.364008:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 552 00000100:00080000:0.0:1789761103.364015:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761103.364018:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761103.364022:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761103.364027:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761103.364034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048702336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761103.364077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761103/real 1789761103] req@ffff9811c2e29f80 x1876700048701824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761103.364109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048701824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761103.364118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761103.364122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761103.364125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048701824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761103.364131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048701952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761103.364149:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761103.364184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761103/real 1789761103] req@ffff9811c2e28a80 x1876700048701952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761103.364192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048701952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761103.364198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761103.364200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761103.364203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048701952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761103.364225:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761103.364238:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761103.364245:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761103/real 1789761103] req@ffff9811c2e28380 x1876700048702080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761103.364252:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048702080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761103.364258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761103.364259:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761103.364261:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761103.364264:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761103.364275:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761103.364280:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761103/real 1789761103] req@ffff9811c2e28e00 x1876700048702208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761103.364287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048702208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761103.364299:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761103.364302:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761103.364304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761103.364308:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761103.364321:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761103.364327:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761103/real 1789761103] req@ffff9811c2e29180 x1876700048702336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761103.364334:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048702336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761103.364339:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761103.364341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761103.364343:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761103.619589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761104.259602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761104.643547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761105.283633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761105.667563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761106.307734:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761106.691596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761107.331618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761107.715596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761108.355620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761108.483606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761108.483617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761108.483642:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761108.483653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 558 00000100:00080000:2.0:1789761108.483666:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761108.483673:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761108.483712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048702464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761108.483735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761108.483742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761108.483744:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761108.483747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 558 00000100:00080000:2.0:1789761108.483751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761108.483756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761108.483768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761108.483784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048702592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761108.483798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761108.483803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761108.483804:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048702464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761108.483805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761108.483814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 558 00000100:00080000:2.0:1789761108.483818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761108.483822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761108.483839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761108.483855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048702720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761108.483880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761108.483884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761108.483886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761108.483890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 558 00000100:00080000:2.0:1789761108.483895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761108.483900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789761108.483907:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761108.483908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761108.483927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048702848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761108.483939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761108.483943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761108.483945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761108.483949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 558 00000100:00080000:2.0:1789761108.483953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761108.483956:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761108.483961:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761108.483961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761108/real 1789761108] req@ffff9811d8639180 x1876700048702464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761108.483974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048702976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761108.483982:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048702464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761108.483993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761108.483996:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761108.484000:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048702464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761108.484030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048702592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761108.484051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761108.484061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761108/real 1789761108] req@ffff9811d863bb80 x1876700048702592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761108.484069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048702592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761108.484076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761108.484078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761108.484081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048702592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761108.484084:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761108.484096:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761108.484102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761108/real 1789761108] req@ffff9811d863b100 x1876700048702720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761108.484109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048702720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761108.484115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761108.484116:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761108.484119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761108.484124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761108.484142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761108.484153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761108/real 1789761108] req@ffff9811d8639c00 x1876700048702848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761108.484174:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048702848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761108.484181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761108.484183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761108.484186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761108.484190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761108.484206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761108.484214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761108/real 1789761108] req@ffff9811d863ad80 x1876700048702976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761108.484221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048702976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761108.484226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761108.484228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761108.484230:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048702976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761108.739530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761109.379588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761109.763601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761110.403676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761110.787512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761111.427630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761111.811590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761112.451620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761112.835538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761113.475607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761113.603586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761113.603595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761113.603599:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761113.603608:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 563 00000100:00080000:2.0:1789761113.603618:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761113.603624:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761113.603658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048703104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761113.603699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761113.603702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761113.603703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761113.603706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 563 00000100:00080000:2.0:1789761113.603709:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761113.603711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761113.603719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761113.603729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048703232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761113.603738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761113.603741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761113.603742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761113.603745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 563 00000100:00080000:2.0:1789761113.603748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761113.603751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761113.603755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761113.603775:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048703104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761113.603784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048703360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761113.603811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761113.603813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761113.603815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761113.603819:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 563 00000100:00080000:2.0:1789761113.603822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761113.603826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761113.603830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761113.603840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048703488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761113.603848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761113.603850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761113.603852:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761113.603855:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 563 00000100:00080000:2.0:1789761113.603858:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761113.603861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761113.603863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761113.603873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048703616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761113.603880:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761113.603938:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761113/real 1789761113] req@ffff9811d863a680 x1876700048703104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761113.603965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048703104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761113.603990:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761113.603994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761113.603998:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048703104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761113.604005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048703232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761113.604025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761113.604035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761113/real 1789761113] req@ffff9811d8639880 x1876700048703232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761113.604043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048703232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761113.604049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761113.604052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761113.604054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048703232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761113.604101:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761113.604125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761113.604138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761113/real 1789761113] req@ffff9811d8639500 x1876700048703360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761113.604147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048703360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761113.604153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761113.604156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761113.604158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761113.604171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761113.604196:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761113.604204:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761113/real 1789761113] req@ffff9811d8639f80 x1876700048703488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761113.604212:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048703488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761113.604218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761113.604220:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761113.604222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761113.604226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761113.604237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761113.604243:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761113/real 1789761113] req@ffff9811d8638a80 x1876700048703616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761113.604250:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048703616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761113.604255:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761113.604257:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761113.604258:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048703616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761113.859541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761114.499641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761114.883595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761115.523563:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761115.907570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761116.547641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761116.931590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761117.571656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761117.955602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761118.595626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761118.723581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761118.723592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761118.723597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761118.723607:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 568 00000100:00080000:0.0:1789761118.723619:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761118.723648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761118.723688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048703744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761118.723712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761118.723716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761118.723717:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761118.723720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 568 00000100:00080000:0.0:1789761118.723723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761118.723726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761118.723735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761118.723744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048703872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761118.723767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761118.723770:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048703744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761118.723771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761118.723775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761118.723782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 568 00000100:00080000:0.0:1789761118.723784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761118.723787:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761118.723790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761118.723799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048704000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761118.723806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761118.723809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761118.723816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761118.723820:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 568 00000100:00080000:0.0:1789761118.723823:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761118.723826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761118.723831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761118.723839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048704128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761118.723846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761118.723849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761118.723850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761118.723852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 568 00000100:00080000:0.0:1789761118.723854:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761118.723859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789761118.723860:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761118.723863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761118.723872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048704256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761118.723909:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761118/real 1789761118] req@ffff9811c2e2aa00 x1876700048703744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761118.723932:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048703744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761118.723940:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761118.723944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761118.723947:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048703744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761118.723970:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048703872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761118.723993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761118.724006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761118/real 1789761118] req@ffff9811c2e2b100 x1876700048703872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761118.724021:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048703872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761118.724030:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761118.724033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761118.724036:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048703872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761118.724041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761118.724057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761118.724065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761118/real 1789761118] req@ffff9811c2e2bb80 x1876700048704000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761118.724073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048704000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761118.724088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761118.724091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761118.724093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761118.724097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761118.724113:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761118.724119:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761118/real 1789761118] req@ffff9811c2e28700 x1876700048704128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761118.724127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048704128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761118.724132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761118.724133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761118.724135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761118.724138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761118.724149:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761118.724154:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761118/real 1789761118] req@ffff9811c2e2a680 x1876700048704256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761118.724161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048704256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761118.724166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761118.724168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761118.724175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761118.979575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761119.619662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761120.003562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761120.643573:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761121.027573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761121.667607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761122.051596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761122.691574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761123.075563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761123.715685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761123.843564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761123.843579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761123.843583:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761123.843599:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 573 00000100:00080000:2.0:1789761123.843612:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761123.843620:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761123.843665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048704384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761123.843688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761123.843692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761123.843693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761123.843696:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 573 00000100:00080000:2.0:1789761123.843699:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761123.843701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761123.843722:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761123.843736:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048704512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761123.843747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761123.843755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761123.843756:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048704384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761123.843774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761123.843781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 573 00000100:00080000:2.0:1789761123.843785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761123.843789:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761123.843795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761123.843805:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048704640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761123.843813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761123.843817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761123.843818:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761123.843821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 573 00000100:00080000:2.0:1789761123.843824:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761123.843829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761123.843830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761123.843842:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761123.843854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048704768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761123.843861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761123.843865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761123.843867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761123.843872:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 573 00000100:00080000:2.0:1789761123.843876:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761123.843880:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761123/real 1789761123] req@ffff9811d863aa00 x1876700048704384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761123.843890:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761123.843905:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761123.843914:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048704384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761123.843927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761123.843928:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048704896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761123.843942:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761123.843945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048704384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761123.843987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048704512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761123.844011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761123.844026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761123/real 1789761123] req@ffff9811d8638000 x1876700048704512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761123.844040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048704512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761123.844050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761123.844054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761123.844058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048704512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761123.844065:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761123.844086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761123.844108:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761123/real 1789761123] req@ffff9811d863b800 x1876700048704640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761123.844116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048704640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761123.844122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761123.844124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761123.844126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761123.844131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761123.844145:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761123.844151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761123/real 1789761123] req@ffff9811d863b480 x1876700048704768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761123.844159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048704768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761123.844163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761123.844165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761123.844167:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761123.844171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761123.844181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761123.844186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761123/real 1789761123] req@ffff9811d8638380 x1876700048704896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761123.844193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048704896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761123.844203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761123.844206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761123.844208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048704896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761124.099587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761124.739665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761125.123586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761125.763629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761126.147594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761126.787560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761127.171630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761127.811613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761128.195551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761128.835575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761128.963587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761128.963598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761128.963603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761128.963614:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 578 00000100:00080000:0.0:1789761128.963626:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761128.963635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761128.963695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048705024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761128.963735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761128.963739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761128.963741:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761128.963745:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 578 00000100:00080000:0.0:1789761128.963749:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761128.963753:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761128.963764:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761128.963777:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048705024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761128.963782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048705152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761128.963794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761128.963797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761128.963799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761128.963802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 578 00000100:00080000:0.0:1789761128.963806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761128.963808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761128.963812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761128.963831:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048705280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761128.963840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761128.963843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761128.963844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761128.963849:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 578 00000100:00080000:0.0:1789761128.963853:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761128.963857:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761128.963861:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761128.963873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048705408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761128.963881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761128.963884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761128.963886:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761128.963889:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 578 00000100:00080000:0.0:1789761128.963892:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761128.963894:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761128.963899:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761128.963901:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048705152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761128.963910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048705536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761128.963933:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761128.963953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761128.963972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761129.219611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761129.859615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761130.243614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761130.883621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761131.267646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761131.907623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761132.035752:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789761132.035797:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761132.035819:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761132.035861:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761132.035870:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761132.035879:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761132.035890:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761132.035917:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761128/real 1789761132] req@ffff9811c2e28700 x1876700048705536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761132.035949:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048705536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761132.035962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761132.035968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761132.035973:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761132.035982:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761128/real 1789761132] req@ffff9811c2e2ad80 x1876700048705152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761132.036017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048705152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761132.036027:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761132.036031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761132.036040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048705152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761132.036049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761128/real 1789761132] req@ffff9811c2e29c00 x1876700048705280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761132.036064:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048705280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761132.036073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761132.036076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761132.036080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761132.036088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761128/real 1789761132] req@ffff9811c2e2a680 x1876700048705408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761132.036100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048705408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761132.036108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761132.036111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761132.036115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761132.036121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761128/real 1789761132] req@ffff9811c2e29880 x1876700048705024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761132.036144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048705024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761132.036152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761132.036155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761132.036159:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048705024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761132.291613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761132.931600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761133.315570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761133.955586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761134.083627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761134.083654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761134.083658:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761134.083668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 583 00000100:00080000:0.0:1789761134.083680:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761134.083688:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761134.083725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048705664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761134.083763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761134.083768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761134.083770:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761134.083779:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 583 00000100:00080000:0.0:1789761134.083783:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761134.083787:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761134.083797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761134.083811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048705792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761134.083823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761134.083826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761134.083828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761134.083832:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 583 00000100:00080000:0.0:1789761134.083836:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761134.083840:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761134.083863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761134.083885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048705920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761134.083905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761134.083913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789761134.083915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048705664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761134.083917:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761134.083936:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 583 00000100:00080000:0.0:1789761134.083942:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761134.083948:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761134.083954:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761134.083970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048706048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761134.083980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761134.083985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761134.083987:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761134.084010:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 583 00000100:00080000:0.0:1789761134.084018:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761134.084023:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761134.084030:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761134.084038:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761134.084047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048706176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761134.084125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761134/real 1789761134] req@ffff9811c2e2bb80 x1876700048705664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761134.084158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048705664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761134.084173:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761134.084179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761134.084185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048705664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761134.084194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048705792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761134.084244:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761134.084263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761134/real 1789761134] req@ffff9811c2e2b100 x1876700048705792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761134.084278:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048705792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761134.084304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761134.084309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761134.084313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048705792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761134.084352:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761134.084380:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761134.084393:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761134/real 1789761134] req@ffff9811c2e2aa00 x1876700048705920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761134.084415:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048705920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761134.084426:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761134.084430:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761134.084434:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048705920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761134.084442:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761134.084524:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761134.084539:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761134/real 1789761134] req@ffff9811c2e29180 x1876700048706048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761134.084554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048706048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761134.084562:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761134.084566:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761134.084577:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761134.084585:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761134.084612:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761134.084625:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761134/real 1789761134] req@ffff9811c2e28e00 x1876700048706176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761134.084659:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048706176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761134.084669:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761134.084673:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761134.084677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761134.339578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761134.979639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761135.363576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761136.003584:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761136.387550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761137.027631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761137.411559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761138.051619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761138.435527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761139.075566:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761139.203567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761139.203575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761139.203578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761139.203586:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 588 00000100:00080000:0.0:1789761139.203594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761139.203614:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761139.203646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048706304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761139.203669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761139.203672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761139.203674:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761139.203678:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 588 00000100:00080000:0.0:1789761139.203682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761139.203687:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761139.203696:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761139.203712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048706432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761139.203715:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048706304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761139.203732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761139.203736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761139.203738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761139.203743:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 588 00000100:00080000:0.0:1789761139.203748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761139.203767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761139.203772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761139.203772:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761139.203787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048706560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761139.203795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761139.203798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761139.203800:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761139.203805:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 588 00000100:00080000:0.0:1789761139.203810:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761139.203812:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761139/real 1789761139] req@ffff9811c2e28380 x1876700048706304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761139.203817:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761139.203829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761139.203835:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048706304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761139.203845:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761139.203846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048706688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761139.203850:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761139.203861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048706304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761139.203862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761139.203868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761139.203870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761139.203874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 588 00000100:00080000:0.0:1789761139.203878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761139.203881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761139.203886:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761139.203887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048706432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761139.203901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048706816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761139.203913:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761139.203934:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761139/real 1789761139] req@ffff9811c2e28a80 x1876700048706432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761139.203944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048706432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761139.203951:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761139.203954:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761139.203956:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048706432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761139.203961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761139.203975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761139.203981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761139/real 1789761139] req@ffff9811c2e29f80 x1876700048706560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761139.203990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048706560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761139.203998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761139.204002:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761139.204005:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761139.204011:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761139.204032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761139.204053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761139/real 1789761139] req@ffff9811c2e29500 x1876700048706688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761139.204063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048706688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761139.204068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761139.204070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761139.204072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761139.204084:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761139.204099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761139.204105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761139/real 1789761139] req@ffff9811c2e2b480 x1876700048706816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761139.204113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048706816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761139.204118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761139.204120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761139.204122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048706816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761139.459529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761140.099603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761140.483560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761141.123619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761141.507626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761142.147548:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761142.531526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761143.171578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761143.555583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761144.195551:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761144.323619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761144.323630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761144.323635:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761144.323649:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 593 00000100:00080000:0.0:1789761144.323664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761144.323674:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761144.323728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048706944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761144.323765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761144.323770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761144.323772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761144.323776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 593 00000100:00080000:0.0:1789761144.323780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761144.323785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761144.323796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761144.323811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048707072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761144.323819:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048706944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761144.323824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761144.323828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761144.323830:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761144.323834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 593 00000100:00080000:0.0:1789761144.323856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761144.323861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761144.323867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761144.323881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048707200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761144.323893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761144.323897:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761144.323900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761144.323903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761144.323920:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 593 00000100:00080000:0.0:1789761144.323927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761144.323932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761144.323939:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761144.323955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048707328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761144.323958:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761144/real 1789761144] req@ffff9811c2e2b800 x1876700048706944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761144.323985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761144.323993:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048706944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761144.324000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761144.324006:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761144.324008:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761144.324014:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 593 00000100:00080000:2.0:1789761144.324014:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761144.324021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048706944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761144.324024:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761144.324032:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761144.324038:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761144.324051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048707072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761144.324054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048707456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761144.324085:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761144.324102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761144/real 1789761144] req@ffff9811c2e28000 x1876700048707072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761144.324116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048707072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761144.324126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761144.324129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761144.324133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048707072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761144.324140:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761144.324164:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761144.324176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761144/real 1789761144] req@ffff9811c2e2a300 x1876700048707200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761144.324187:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048707200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761144.324203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761144.324206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761144.324208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761144.324213:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761144.324231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761144.324238:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761144/real 1789761144] req@ffff9811c2e2b480 x1876700048707328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761144.324246:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048707328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761144.324252:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761144.324254:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761144.324256:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761144.324267:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761144.324278:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761144.324284:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761144/real 1789761144] req@ffff9811c2e29500 x1876700048707456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761144.324292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048707456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761144.324300:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761144.324303:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761144.324315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761144.579616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761145.219574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761145.603560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761146.243555:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761146.627683:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761147.267604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761147.651555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761148.291569:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761148.675609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761149.315630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761149.443585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761149.443596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761149.443600:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761149.443610:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 599 00000100:00080000:0.0:1789761149.443623:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761149.443630:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761149.443674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048707584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761149.443700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761149.443703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761149.443705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761149.443708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 599 00000100:00080000:0.0:1789761149.443710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761149.443712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761149.443721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761149.443731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048707712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761149.443746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761149.443749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789761149.443758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048707584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761149.443772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761149.443776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 599 00000100:00080000:0.0:1789761149.443780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761149.443783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761149.443787:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761149.443796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048707840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761149.443804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761149.443807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761149.443808:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761149.443810:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 599 00000100:00080000:0.0:1789761149.443813:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761149.443815:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761149.443819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761149.443826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048707968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761149.443832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761149.443834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761149.443838:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761149.443839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761149.443842:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 599 00000100:00080000:0.0:1789761149.443845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761149.443848:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761149.443851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761149.443865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048708096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761149.443887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761149/real 1789761149] req@ffff9811c2e29f80 x1876700048707584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761149.443907:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048707584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761149.443919:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761149.443924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761149.443929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048707584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761149.443957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048707712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761149.443978:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761149.443988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761149/real 1789761149] req@ffff9811c2e28a80 x1876700048707712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761149.443997:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048707712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761149.444003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761149.444006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761149.444008:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048707712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761149.444011:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761149.444033:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761149.444042:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761149/real 1789761149] req@ffff9811c2e28380 x1876700048707840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761149.444056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048707840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761149.444064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761149.444068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761149.444071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761149.444078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761149.444101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761149.444113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761149/real 1789761149] req@ffff9811c2e28e00 x1876700048707968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761149.444127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048707968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761149.444136:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761149.444139:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761149.444143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048707968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761149.444148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761149.444165:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761149.444177:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761149/real 1789761149] req@ffff9811c2e29180 x1876700048708096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761149.444205:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048708096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761149.444214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761149.444218:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761149.444222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761149.699554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761150.339594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761150.723566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761151.363619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761151.747581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761152.387642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761152.771626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761153.411654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761153.795593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761154.435602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761154.563569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761154.563577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761154.563580:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761154.563589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 604 00000100:00080000:0.0:1789761154.563599:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761154.563605:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761154.563646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048708224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761154.563665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761154.563668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761154.563669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761154.563672:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 604 00000100:00080000:0.0:1789761154.563675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761154.563677:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761154.563686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761154.563696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048708352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761154.563700:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048708224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761154.563714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761154.563726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761154.563728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761154.563732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 604 00000100:00080000:0.0:1789761154.563735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761154.563738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761154.563743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761154.563752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048708480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761154.563759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761154.563762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761154.563763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761154.563766:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 604 00000100:00080000:0.0:1789761154.563771:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761154.563771:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761154.563777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761154.563781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761154.563789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048708608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761154.563795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761154.563798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761154.563799:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761154.563802:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 604 00000100:00080000:0.0:1789761154.563805:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761154.563808:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761154/real 1789761154] req@ffff9811c2e2aa00 x1876700048708224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761154.563811:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761154.563822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761154.563828:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048708224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761154.563832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048708736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761154.563843:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761154.563849:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761154.563853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048708224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761154.563874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048708352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761154.563890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761154.563899:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761154/real 1789761154] req@ffff9811c2e2b100 x1876700048708352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761154.563908:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048708352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761154.563929:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761154.563933:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761154.563937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048708352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761154.563944:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761154.563975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761154.563984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761154/real 1789761154] req@ffff9811c2e2bb80 x1876700048708480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761154.563992:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048708480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761154.563998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761154.564000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761154.564002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761154.564006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761154.564018:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761154.564024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761154/real 1789761154] req@ffff9811c2e29880 x1876700048708608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761154.564032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048708608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761154.564036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761154.564038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761154.564046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761154.564050:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761154.564063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761154.564069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761154/real 1789761154] req@ffff9811c2e2a680 x1876700048708736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761154.564077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048708736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761154.564082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761154.564083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761154.564085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048708736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761154.819560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761155.459628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761155.843622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761156.483681:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761156.867614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761157.507600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761157.891557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761158.531628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761158.915574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761159.555607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761159.683607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761159.683618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761159.683622:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761159.683634:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 609 00000100:00080000:3.0:1789761159.683646:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761159.683654:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761159.683698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048708864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761159.683724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761159.683727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761159.683728:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761159.683757:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 609 00000100:00080000:3.0:1789761159.683761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761159.683764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761159.683774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761159.683789:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048708864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761159.683789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048708992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761159.683808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761159.683811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761159.683812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761159.683817:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 609 00000100:00080000:3.0:1789761159.683820:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761159.683823:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761159.683829:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761159.683841:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048709120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761159.683848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761159.683852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761159.683853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761159.683856:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 609 00000800:00000100:2.0:1789761159.683863:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761159.683863:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761159.683874:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761159.683880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761159.683893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048709248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761159.683905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761159.683907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761159.683909:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761159.683913:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 609 00000100:00080000:3.0:1789761159.683916:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761159.683919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761159/real 1789761159] req@ffff9811e1a23800 x1876700048708864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761159.683922:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761159.683932:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761159.683953:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048708864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761159.683963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761159.683963:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048709376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761159.683975:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761159.683985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048708864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761159.684023:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048708992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761159.684046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761159.684069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761159/real 1789761159] req@ffff9811e1a20000 x1876700048708992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761159.684077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048708992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761159.684083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761159.684086:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761159.684088:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048708992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761159.684092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761159.684105:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761159.684111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761159/real 1789761159] req@ffff9811e1a23480 x1876700048709120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761159.684118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048709120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761159.684123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761159.684125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761159.684127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761159.684131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761159.684141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761159.684147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761159/real 1789761159] req@ffff9811e1a23100 x1876700048709248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761159.684153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048709248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761159.684163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761159.684165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761159.684168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761159.684171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761159.684184:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761159.684189:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761159/real 1789761159] req@ffff9811e1a20700 x1876700048709376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761159.684197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048709376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761159.684202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761159.684204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761159.684205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761159.939591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761160.579652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761160.963569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761161.603612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761161.987624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761162.627662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761163.011545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761163.651642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761164.035549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761164.675652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761164.803578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761164.803587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761164.803592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761164.803603:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 614 00000100:00080000:2.0:1789761164.803616:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761164.803624:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761164.803661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048709504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761164.803692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761164.803702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761164.803704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761164.803708:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 614 00000100:00080000:2.0:1789761164.803711:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761164.803715:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761164.803725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761164.803738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048709632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761164.803747:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048709504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761164.803753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761164.803757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761164.803759:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761164.803767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 614 00000100:00080000:2.0:1789761164.803771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761164.803776:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761164.803780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761164.803791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048709760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761164.803799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761164.803801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761164.803803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761164.803805:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 614 00000100:00080000:2.0:1789761164.803807:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761164.803810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761164.803821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761164.803830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048709888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761164.803838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761164.803841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761164.803842:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761164.803847:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761164.803848:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 614 00000100:00080000:2.0:1789761164.803854:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761164.803857:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761164.803860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761164.803876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048710016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761164.803897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761164/real 1789761164] req@ffff9811d8638e00 x1876700048709504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761164.803916:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048709504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761164.803925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761164.803928:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761164.803931:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048709504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761164.803955:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048709632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761164.803973:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761164.803982:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761164/real 1789761164] req@ffff9811d8638700 x1876700048709632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761164.803990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048709632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761164.803996:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761164.803999:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761164.804001:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048709632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761164.804004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761164.804015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761164.804021:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761164/real 1789761164] req@ffff9811d863a300 x1876700048709760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761164.804042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048709760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761164.804051:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761164.804054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761164.804058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761164.804064:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761164.804088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761164.804100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761164/real 1789761164] req@ffff9811d8638380 x1876700048709888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761164.804109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048709888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761164.804115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761164.804117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761164.804119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048709888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761164.804124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761164.804139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761164.804145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761164/real 1789761164] req@ffff9811d863b480 x1876700048710016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761164.804152:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048710016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761164.804171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761164.804175:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761164.804180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761165.059527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761165.699678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761166.083609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761166.723625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761167.107542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761167.747615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761168.131548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761168.771632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761169.155537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761169.795666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761169.923588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761169.923603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761169.923609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761169.923623:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 619 00000100:00080000:0.0:1789761169.923636:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761169.923667:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761169.923716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048710144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761169.923750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761169.923753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761169.923755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761169.923759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 619 00000100:00080000:0.0:1789761169.923763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761169.923767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761169.923777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761169.923790:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048710144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761169.923791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048710272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761169.923808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761169.923810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761169.923812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761169.923815:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 619 00000100:00080000:0.0:1789761169.923817:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761169.923820:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761169.923823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761169.923830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048710400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761169.923837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761169.923839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761169.923846:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761169.923850:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 619 00000100:00080000:0.0:1789761169.923870:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761169.923875:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761169.923881:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761169.923887:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761169.923901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048710528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761169.923913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761169.923917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761169.923919:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761169.923924:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 619 00000100:00080000:0.0:1789761169.923929:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761169.923937:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761169/real 1789761169] req@ffff9811c2e29c00 x1876700048710144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761169.923938:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761169.923945:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761169.923960:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048710144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761169.923962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048710656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761169.923986:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761169.923992:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761169.923995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048710144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761169.924019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048710272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761169.924039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761169.924053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761169/real 1789761169] req@ffff9811c2e2ad80 x1876700048710272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761169.924067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048710272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761169.924090:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761169.924093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761169.924096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048710272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761169.924100:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761169.924117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761169.924126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761169/real 1789761169] req@ffff9811c2e28700 x1876700048710400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761169.924133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048710400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761169.924139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761169.924141:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761169.924143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761169.924147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761169.924158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761169.924163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761169/real 1789761169] req@ffff9811c2e2a680 x1876700048710528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761169.924171:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048710528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761169.924176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761169.924178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761169.924186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761169.924190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761169.924203:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761169.924210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761169/real 1789761169] req@ffff9811c2e29880 x1876700048710656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761169.924217:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048710656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761169.924222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761169.924224:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761169.924226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048710656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761170.179574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761170.819613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761171.203607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761171.843596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761172.227576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761172.867636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761173.251589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761173.891669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761174.275610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761174.915612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761175.043600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761175.043611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761175.043615:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761175.043629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 624 00000100:00080000:2.0:1789761175.043643:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761175.043653:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761175.043718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048710784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761175.043755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761175.043760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761175.043762:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761175.043767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 624 00000100:00080000:2.0:1789761175.043795:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761175.043802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761175.043814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761175.043819:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048710784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761175.043833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048710912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761175.043849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761175.043853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761175.043856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761175.043861:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 624 00000100:00080000:2.0:1789761175.043865:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761175.043870:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761175.043876:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761175.043889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048711040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761175.043900:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761175.043904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761175.043910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761175.043912:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761175.043918:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 624 00000100:00080000:2.0:1789761175.043924:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761175.043929:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761175.043936:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761175.043948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761175/real 1789761175] req@ffff9811d863b800 x1876700048710784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761175.043953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048711168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761175.043973:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048710784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761175.044002:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761175.044003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761175.044009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761175.044010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789761175.044012:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761175.044016:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048710784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761175.044021:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 624 00000100:00080000:2.0:1789761175.044026:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761175.044030:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761175.044035:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761175.044045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048710912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761175.044047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048711296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761175.044078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761175.044091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761175/real 1789761175] req@ffff9811d8638000 x1876700048710912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761175.044100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048710912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761175.044106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761175.044108:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761175.044111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048710912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761175.044114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761175.044126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761175.044132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761175/real 1789761175] req@ffff9811d863aa00 x1876700048711040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761175.044139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048711040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761175.044145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761175.044146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761175.044148:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761175.044152:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761175.044166:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761175.044176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761175/real 1789761175] req@ffff9811d8638a80 x1876700048711168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761175.044204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048711168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761175.044213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761175.044217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761175.044221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761175.044236:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761175.044252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761175.044260:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761175/real 1789761175] req@ffff9811d8639f80 x1876700048711296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761175.044268:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048711296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761175.044273:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761175.044276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761175.044278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761175.299601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761175.939617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761176.323551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761176.963582:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761177.347577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761177.987624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761178.371659:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761179.011619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761179.395573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761180.035625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761180.163533:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761180.163541:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761180.163544:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761180.163553:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 629 00000100:00080000:2.0:1789761180.163561:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761180.163567:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761180.163599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048711424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761180.163621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761180.163623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761180.163625:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761180.163627:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 629 00000100:00080000:2.0:1789761180.163629:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761180.163631:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761180.163656:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761180.163661:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048711424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761180.163668:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048711552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761180.163679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761180.163681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761180.163683:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761180.163685:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 629 00000100:00080000:2.0:1789761180.163688:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761180.163691:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761180.163694:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761180.163702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048711680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761180.163708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761180.163710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761180.163712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761180.163714:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 629 00000100:00080000:2.0:1789761180.163717:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761180.163719:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761180.163722:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761180.163723:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761180.163732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048711808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761180.163740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761180.163743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761180.163749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761180.163752:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 629 00000100:00080000:2.0:1789761180.163756:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761180.163759:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761180.163763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761180.163765:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761180/real 1789761180] req@ffff9811d8639500 x1876700048711424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761180.163774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048711936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761180.163798:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048711424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761180.163820:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761180.163825:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761180.163828:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048711424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761180.163848:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048711552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761180.163867:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761180.163876:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761180/real 1789761180] req@ffff9811d8639880 x1876700048711552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761180.163885:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048711552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761180.163890:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761180.163893:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761180.163895:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048711552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761180.163899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761180.163909:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761180.163915:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761180/real 1789761180] req@ffff9811d863a680 x1876700048711680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761180.163922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048711680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761180.163927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761180.163931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761180.163934:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761180.163947:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761180.163970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761180.163983:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761180/real 1789761180] req@ffff9811d863ad80 x1876700048711808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761180.163995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048711808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761180.164003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761180.164006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761180.164010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761180.164016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761180.164036:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761180.164045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761180/real 1789761180] req@ffff9811d8639c00 x1876700048711936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761180.164053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048711936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761180.164058:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761180.164060:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761180.164062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048711936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761180.419579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761181.059574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761181.443600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761182.083621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761182.467581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761183.107562:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761183.491593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761184.131610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761184.515527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761185.155614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761185.283609:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761185.283619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761185.283623:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761185.283634:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 634 00000100:00080000:0.0:1789761185.283647:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761185.283656:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761185.283693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048712064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761185.283720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761185.283724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761185.283726:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761185.283730:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 634 00000100:00080000:0.0:1789761185.283733:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761185.283737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761185.283747:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761185.283761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048712192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761185.283773:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048712064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761185.283822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761185.283826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761185.283828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761185.283833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 634 00000100:00080000:0.0:1789761185.283838:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761185.283843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761185.283849:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761185.283852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761185.283869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048712320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761185.283879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761185.283883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761185.283885:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761185.283890:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 634 00000100:00080000:0.0:1789761185.283894:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761185.283899:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761185/real 1789761185] req@ffff9811c2e2bb80 x1876700048712064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761185.283903:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761185.283912:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761185.283927:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048712064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761185.283931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048712448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761185.283948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761185.283957:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789761185.283958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761185.283964:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048712064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761185.283965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761185.283969:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761185.283976:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 634 00000100:00080000:0.0:1789761185.283989:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761185.283993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048712192:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789761185.283996:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761185.284006:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761185.284021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048712576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761185.284031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761185.284053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761185/real 1789761185] req@ffff9811c2e2b100 x1876700048712192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761185.284069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048712192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761185.284077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761185.284081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761185.284084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048712192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761185.284091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761185.284112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761185.284124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761185/real 1789761185] req@ffff9811c2e2aa00 x1876700048712320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761185.284149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048712320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761185.284157:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761185.284160:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761185.284164:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761185.284170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761185.284190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761185.284203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761185/real 1789761185] req@ffff9811c2e29180 x1876700048712448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761185.284227:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048712448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761185.284235:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761185.284238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761185.284242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761185.284276:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761185.284306:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761185.284318:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761185/real 1789761185] req@ffff9811c2e28e00 x1876700048712576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761185.284330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048712576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761185.284349:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761185.284352:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761185.284355:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761185.539641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761186.179584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761186.563542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761187.203652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761187.587647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761188.227637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761188.611553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761189.251678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761189.635650:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761190.275590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761190.403627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761190.403638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761190.403641:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761190.403650:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 639 00000100:00080000:2.0:1789761190.403660:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761190.403666:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761190.403697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048712704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761190.403722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761190.403724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761190.403726:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761190.403728:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 639 00000100:00080000:2.0:1789761190.403730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761190.403732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761190.403740:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761190.403761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048712832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761190.403775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761190.403778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761190.403782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761190.403785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048712704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761190.403788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 639 00000100:00080000:2.0:1789761190.403802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761190.403806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761190.403812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761190.403824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048712960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761190.403832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761190.403834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761190.403835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761190.403838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 639 00000100:00080000:2.0:1789761190.403840:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761190.403843:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761190.403846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761190.403853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048713088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761190.403858:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761190.403865:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761190.403868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761190.403870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761190.403878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 639 00000100:00080000:2.0:1789761190.403882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761190.403885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761190.403888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761190.403896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048713216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761190.403908:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761190/real 1789761190] req@ffff9811d863b100 x1876700048712704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761190.403926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048712704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761190.403938:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761190.403943:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761190.403947:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048712704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761190.403972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048712832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761190.403999:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761190.404013:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761190/real 1789761190] req@ffff9811d863bb80 x1876700048712832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761190.404026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048712832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761190.404036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761190.404040:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761190.404044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048712832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761190.404058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761190.404082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761190.404095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761190/real 1789761190] req@ffff9811d8639180 x1876700048712960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761190.404109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048712960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761190.404117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761190.404120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761190.404124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048712960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761190.404130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761190.404150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761190.404163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761190/real 1789761190] req@ffff9811d8639c00 x1876700048713088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761190.404175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048713088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761190.404183:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761190.404186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761190.404190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761190.404196:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761190.404231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761190.404244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761190/real 1789761190] req@ffff9811d863ad80 x1876700048713216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761190.404265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048713216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761190.404274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761190.404277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761190.404281:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761190.659605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761191.299643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761191.683607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761192.323614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761192.707569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761193.347606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761193.731559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761194.371593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761194.755622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761195.395647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761195.523580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761195.523591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761195.523596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761195.523607:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 645 00000100:00080000:2.0:1789761195.523619:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761195.523626:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761195.523670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048713344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761195.523696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761195.523699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761195.523700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761195.523703:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 645 00000100:00080000:2.0:1789761195.523706:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761195.523708:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761195.523717:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761195.523727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048713472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761195.523763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761195.523764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048713344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761195.523768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761195.523771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761195.523783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 645 00000100:00080000:2.0:1789761195.523787:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761195.523791:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761195.523795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761195.523807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048713600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761195.523815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761195.523817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761195.523819:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761195.523822:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 645 00000100:00080000:2.0:1789761195.523825:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761195.523827:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761195.523830:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761195.523838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048713728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761195.523846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761195.523849:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761195.523850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761195.523853:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 645 00000100:00080000:2.0:1789761195.523860:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761195.523864:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761195.523870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761195.523882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048713856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761195.523891:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048713472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761195.523924:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761195.523947:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761195.523965:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761195.779623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761196.419587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761196.803626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761197.443653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761197.827544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761198.467572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761198.595691:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789761198.595727:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761198.595749:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761198.595782:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761198.595793:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789761198.595823:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761195/real 1789761198] req@ffff9811d8639880 x1876700048713472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789761198.595824:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761198.595840:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789761198.595856:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048713472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761198.595870:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761198.595881:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761198.595886:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048713472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761198.595895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761195/real 1789761198] req@ffff9811d8639500 x1876700048713600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761198.595924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048713600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761198.595941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761198.595945:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761198.595949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761198.595956:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761195/real 1789761198] req@ffff9811d8639f80 x1876700048713728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761198.595970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048713728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761198.595975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761198.595978:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761198.595981:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761198.595988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761195/real 1789761198] req@ffff9811d863a680 x1876700048713344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761198.596001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048713344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761198.596009:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761198.596013:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761198.596016:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048713344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761198.596043:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761195/real 1789761198] req@ffff9811d8638a80 x1876700048713856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761198.596064:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048713856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761198.596073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761198.596076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761198.596079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048713856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761198.851570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761199.491702:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761199.875584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761200.515585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761200.643773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761200.643785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761200.643790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761200.643801:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 650 00000100:00080000:0.0:1789761200.643815:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761200.643823:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761200.643868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048713984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761200.643899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761200.643903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761200.643905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761200.643911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 650 00000100:00080000:0.0:1789761200.643916:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761200.643922:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761200.643936:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761200.643962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048714112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761200.643978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761200.643982:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048713984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761200.643984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761200.643990:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761200.644022:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 650 00000100:00080000:0.0:1789761200.644030:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761200.644036:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761200.644043:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761200.644062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048714240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761200.644074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761200.644078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761200.644081:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761200.644085:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 650 00000100:00080000:0.0:1789761200.644089:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761200.644093:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761200.644100:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761200.644101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761200.644118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048714368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761200.644130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761200.644134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761200.644150:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761200.644160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 650 00000100:00080000:0.0:1789761200.644164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761200.644169:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761200.644176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761200.644184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761200/real 1789761200] req@ffff9811c2e28380 x1876700048713984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761200.644192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048714496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761200.644214:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048713984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761200.644229:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761200.644235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761200.644241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048713984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761200.644285:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048714112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761200.644319:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761200.644339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761200/real 1789761200] req@ffff9811c2e28a80 x1876700048714112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761200.644351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048714112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761200.644377:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761200.644383:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761200.644387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048714112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761200.644395:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761200.644423:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761200.644440:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761200/real 1789761200] req@ffff9811c2e29f80 x1876700048714240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761200.644453:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048714240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761200.644498:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761200.644503:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761200.644507:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761200.644514:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761200.644539:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761200.644552:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761200/real 1789761200] req@ffff9811c2e29500 x1876700048714368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761200.644565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048714368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761200.644574:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761200.644578:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761200.644590:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761200.644600:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761200.644626:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761200.644638:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761200/real 1789761200] req@ffff9811c2e2b480 x1876700048714496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761200.644650:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048714496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761200.644659:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761200.644663:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761200.644666:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761200.899565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761201.539592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761201.923544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761202.563611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761202.947539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761203.587626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761203.971536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761204.611661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761204.995546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761205.635642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761205.763670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761205.763679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761205.763684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761205.763693:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 655 00000100:00080000:0.0:1789761205.763705:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761205.763714:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761205.763756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048714624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761205.763779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761205.763782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761205.763784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761205.763788:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 655 00000100:00080000:0.0:1789761205.763793:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761205.763796:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761205.763806:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761205.763822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048714752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761205.763822:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048714624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761205.763855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761205.763858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761205.763875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761205.763882:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 655 00000100:00080000:0.0:1789761205.763894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761205.763899:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761205.763906:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761205.763922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048714880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761205.763929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761205.763932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761205.763934:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761205.763943:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 655 00000800:00000100:2.0:1789761205.763945:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761205.763950:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761205.763956:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761205.763961:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761205.763977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048715008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761205.763985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761205.763988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761205.763990:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761205.763994:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 655 00000100:00080000:0.0:1789761205.763998:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761205.764003:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761205.764008:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761205.764021:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761205/real 1789761205] req@ffff9811c2e2a300 x1876700048714624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761205.764031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048715136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761205.764111:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048714624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761205.764130:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761205.764135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761205.764141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048714624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761205.764171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048714752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761205.764198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761205.764212:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761205/real 1789761205] req@ffff9811c2e28000 x1876700048714752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761205.764225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048714752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761205.764234:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761205.764238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761205.764255:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048714752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761205.764262:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761205.764301:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761205.764317:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761205/real 1789761205] req@ffff9811c2e2b800 x1876700048714880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761205.764330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048714880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761205.764338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761205.764341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761205.764345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048714880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761205.764353:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761205.764373:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761205.764383:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761205/real 1789761205] req@ffff9811c2e2b480 x1876700048715008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761205.764399:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048715008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761205.764407:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761205.764411:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761205.764414:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761205.764421:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761205.764443:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761205.764485:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761205/real 1789761205] req@ffff9811c2e29500 x1876700048715136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761205.764501:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048715136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761205.764510:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761205.764515:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761205.764518:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761206.019556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761206.659620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761207.043556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761207.683630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761208.067526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761208.707638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761209.091546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761209.731635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761210.115517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761210.755636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761210.883574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761210.883585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761210.883589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761210.883601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 660 00000100:00080000:2.0:1789761210.883615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761210.883624:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761210.883675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048715264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761210.883710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761210.883715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761210.883717:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761210.883722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 660 00000100:00080000:2.0:1789761210.883728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761210.883734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761210.883748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761210.883756:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048715264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761210.883772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048715392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761210.883807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761210.883810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761210.883812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761210.883816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 660 00000100:00080000:2.0:1789761210.883821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761210.883828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761210.883836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761210.883838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761210.883854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048715520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761210.883863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761210.883866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761210.883868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761210.883872:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 660 00000100:00080000:2.0:1789761210.883876:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761210.883880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761210.883884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761210.883886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761210/real 1789761210] req@ffff9811d863aa00 x1876700048715264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761210.883895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048715648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761210.883906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761210.883909:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048715264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761210.883916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761210.883921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761210.883935:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 660 00000100:00080000:2.0:1789761210.883945:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761210.883952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761210.883965:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761210.883975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761210.883980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761210.883991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048715264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761210.883992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048715776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761210.884020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048715392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761210.884044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761210.884054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761210/real 1789761210] req@ffff9811d8638000 x1876700048715392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761210.884130:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048715392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761210.884137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761210.884140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761210.884142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048715392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761210.884147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761210.884163:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761210.884171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761210/real 1789761210] req@ffff9811d863b800 x1876700048715520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761210.884179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048715520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761210.884184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761210.884186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761210.884188:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761210.884199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761210.884246:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761210.884255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761210/real 1789761210] req@ffff9811d863b480 x1876700048715648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761210.884264:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048715648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761210.884286:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761210.884288:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761210.884290:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761210.884294:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761210.884306:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761210.884312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761210/real 1789761210] req@ffff9811d8638380 x1876700048715776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761210.884318:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048715776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761210.884323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761210.884325:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761210.884327:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048715776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761211.139536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761211.779616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761212.163566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761212.803626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761213.187572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761213.827612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761214.211559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761214.851570:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761215.235591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761215.875588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761216.003589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761216.003600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761216.003605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761216.003616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 665 00000100:00080000:2.0:1789761216.003628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761216.003636:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761216.003677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048715904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761216.003704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761216.003707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761216.003708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761216.003711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 665 00000100:00080000:2.0:1789761216.003713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761216.003716:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761216.003724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761216.003736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048716032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761216.003746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761216.003760:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048715904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761216.003769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761216.003771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761216.003776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 665 00000100:00080000:2.0:1789761216.003779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761216.003782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761216.003786:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761216.003797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048716160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761216.003810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761216.003814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761216.003816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761216.003822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 665 00000100:00080000:2.0:1789761216.003826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761216.003831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761216.003835:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761216.003840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761216.003857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048716288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761216.003864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761216.003867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761216.003868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761216.003871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 665 00000100:00080000:2.0:1789761216.003874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761216.003877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761216.003887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761216.003891:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761216/real 1789761216] req@ffff9811d863a300 x1876700048715904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761216.003900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048716416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761216.003936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048715904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761216.003948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761216.003954:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761216.003959:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048715904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761216.003970:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048716032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761216.003997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761216.004007:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761216/real 1789761216] req@ffff9811d8638700 x1876700048716032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761216.004017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048716032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761216.004037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761216.004041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761216.004044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048716032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761216.004082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761216.004100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761216.004109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761216/real 1789761216] req@ffff9811d8638e00 x1876700048716160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761216.004118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048716160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761216.004133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761216.004136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761216.004138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761216.004142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761216.004157:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761216.004164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761216/real 1789761216] req@ffff9811d8638380 x1876700048716288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761216.004172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048716288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761216.004177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761216.004179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761216.004181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761216.004184:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761216.004195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761216.004201:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761216/real 1789761216] req@ffff9811d863b480 x1876700048716416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761216.004209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048716416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761216.004213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761216.004215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761216.004222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761216.259631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761216.899629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761217.283563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761217.923661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761218.307594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761218.947629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761219.331538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761219.971571:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761220.355569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761220.995612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761221.123554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761221.123562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761221.123566:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761221.123574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 670 00000100:00080000:2.0:1789761221.123584:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761221.123590:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761221.123622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048716544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761221.123647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761221.123650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761221.123652:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761221.123656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 670 00000100:00080000:2.0:1789761221.123660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761221.123664:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761221.123676:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761221.123689:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048716544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761221.123692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048716672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761221.123707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761221.123711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761221.123713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761221.123717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 670 00000100:00080000:2.0:1789761221.123735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761221.123738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761221.123743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761221.123754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048716800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761221.123762:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761221.123765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761221.123770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761221.123772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761221.123777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 670 00000100:00080000:2.0:1789761221.123781:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761221.123785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761221.123791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761221.123805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048716928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761221.123815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761221.123819:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761221/real 1789761221] req@ffff9811d863b800 x1876700048716544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761221.123821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761221.123825:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761221.123833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 670 00000100:00080000:2.0:1789761221.123840:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761221.123845:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761221.123850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761221.123851:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048716544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761221.123867:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761221.123867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048717056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761221.123878:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761221.123884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048716544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761221.123916:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048716672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761221.123943:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761221.123958:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761221/real 1789761221] req@ffff9811d8638000 x1876700048716672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761221.123972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048716672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761221.123981:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761221.123998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761221.124002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048716672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761221.124019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761221.124043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761221.124055:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761221/real 1789761221] req@ffff9811d863aa00 x1876700048716800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761221.124067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048716800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761221.124076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761221.124081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761221.124084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761221.124091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761221.124112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761221.124123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761221/real 1789761221] req@ffff9811d8638a80 x1876700048716928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761221.124136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048716928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761221.124145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761221.124149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761221.124152:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048716928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761221.124159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761221.124178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761221.124189:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761221/real 1789761221] req@ffff9811d863a680 x1876700048717056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761221.124209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048717056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761221.124218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761221.124222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761221.124226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761221.379530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761222.019650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761222.403558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761223.043598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761223.427542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761224.067668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761224.451569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761225.091646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761225.475530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761226.115615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761226.243582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761226.243593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761226.243597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761226.243607:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 675 00000100:00080000:3.0:1789761226.243634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761226.243643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761226.243676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048717184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761226.243708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761226.243712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761226.243714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761226.243718:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 675 00000100:00080000:3.0:1789761226.243723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761226.243727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761226.243737:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761226.243753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048717312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761226.243758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048717184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761226.243784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761226.243788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761226.243791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761226.243797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 675 00000100:00080000:3.0:1789761226.243802:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761226.243807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761226.243813:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761226.243824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761226.243829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048717440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761226.243842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761226.243852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761226.243855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761226.243859:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 675 00000100:00080000:3.0:1789761226.243865:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761226.243871:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761226/real 1789761226] req@ffff9811e1a21880 x1876700048717184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761226.243874:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761226.243884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761226.243898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048717184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761226.243903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048717568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761226.243923:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761226.243925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761226.243931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761226.243933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761226.243937:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761226.243941:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 675 00000100:00080000:3.0:1789761226.243944:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761226.243962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048717184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761226.243965:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761226.243976:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761226.243990:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048717696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761226.243994:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048717312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761226.244031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761226.244048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761226/real 1789761226] req@ffff9811e1a21180 x1876700048717312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761226.244063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048717312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761226.244081:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761226.244085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761226.244090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048717312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761226.244096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761226.244118:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761226.244129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761226/real 1789761226] req@ffff9811e1a22a00 x1876700048717440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761226.244142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048717440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761226.244150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761226.244154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761226.244157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761226.244163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761226.244183:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761226.244194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761226/real 1789761226] req@ffff9811e1a21c00 x1876700048717568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761226.244207:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048717568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761226.244216:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761226.244228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761226.244238:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761226.244254:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761226.244273:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761226.244283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761226/real 1789761226] req@ffff9811e1a22300 x1876700048717696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761226.244295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048717696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761226.244304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761226.244307:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761226.244310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048717696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761226.499599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761227.139651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761227.523533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761228.163662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761228.547592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761229.187665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761229.571532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761230.211658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761230.595538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761231.235626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761231.363590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761231.363599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761231.363604:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761231.363614:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 680 00000100:00080000:2.0:1789761231.363625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761231.363633:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761231.363675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048717824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761231.363703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761231.363706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761231.363707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761231.363710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 680 00000100:00080000:2.0:1789761231.363713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761231.363715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761231.363723:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761231.363734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048717952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761231.363743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761231.363746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761231.363750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761231.363751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048717824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761231.363755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 680 00000100:00080000:2.0:1789761231.363761:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761231.363764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761231.363789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761231.363803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048718080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761231.363811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761231.363814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761231.363815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761231.363818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 680 00000100:00080000:2.0:1789761231.363822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761231.363825:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761231.363828:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761231.363834:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761231.363838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048718208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761231.363846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761231.363849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761231.363850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761231.363853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 680 00000100:00080000:2.0:1789761231.363855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761231.363857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761231.363860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761231.363868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048718336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761231.363888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761231/real 1789761231] req@ffff9811d8639f80 x1876700048717824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761231.363905:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048717824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761231.363927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761231.363930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761231.363933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048717824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761231.363966:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048717952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761231.363985:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761231.363995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761231/real 1789761231] req@ffff9811d8639500 x1876700048717952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761231.364003:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048717952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761231.364009:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761231.364012:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761231.364014:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048717952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761231.364017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761231.364029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761231.364035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761231/real 1789761231] req@ffff9811d8639880 x1876700048718080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761231.364042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048718080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761231.364047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761231.364049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761231.364051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761231.364054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761231.364064:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761231.364070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761231/real 1789761231] req@ffff9811d863ad80 x1876700048718208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761231.364084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048718208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761231.364089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761231.364092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761231.364094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761231.364097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761231.364109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761231.364114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761231/real 1789761231] req@ffff9811d8639c00 x1876700048718336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761231.364122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048718336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761231.364127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761231.364128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761231.364130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761231.619572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761232.259614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761232.643585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761233.283631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761233.667585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761234.307688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761234.691527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761235.331612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761235.715576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761236.355625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761236.483593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761236.483604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761236.483609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761236.483624:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 686 00000100:00080000:0.0:1789761236.483636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761236.483646:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761236.483687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048718464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761236.483712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761236.483715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761236.483717:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761236.483720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 686 00000100:00080000:0.0:1789761236.483726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761236.483731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761236.483741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761236.483762:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048718464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761236.483772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048718592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761236.483790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761236.483793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761236.483795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761236.483798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 686 00000100:00080000:0.0:1789761236.483804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761236.483808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761236.483813:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761236.483824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048718720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761236.483831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761236.483834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761236.483835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761236.483838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 686 00000100:00080000:0.0:1789761236.483845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761236.483847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761236.483850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761236.483861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048718848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761236.483862:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761236.483883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761236.483886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761236.483887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761236.483898:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 686 00000100:00080000:0.0:1789761236.483902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761236.483906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761236.483911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761236.483922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048718976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761236.483949:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761236/real 1789761236] req@ffff9811c2e29f80 x1876700048718464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761236.483968:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048718464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761236.483977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761236.483981:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761236.483984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048718464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761236.484010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048718592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761236.484031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761236.484042:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761236/real 1789761236] req@ffff9811c2e28a80 x1876700048718592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761236.484052:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048718592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761236.484058:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761236.484062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761236.484065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048718592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761236.484071:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761236.484086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761236.484093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761236/real 1789761236] req@ffff9811c2e28380 x1876700048718720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761236.484110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048718720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761236.484126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761236.484129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761236.484133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761236.484138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761236.484158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761236.484174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761236/real 1789761236] req@ffff9811c2e28e00 x1876700048718848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761236.484189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048718848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761236.484197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761236.484199:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761236.484202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761236.484205:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761236.484225:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761236.484248:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761236/real 1789761236] req@ffff9811c2e29180 x1876700048718976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761236.484265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048718976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761236.484274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761236.484277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761236.484289:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048718976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761236.739552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761237.379634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761237.763572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761238.403652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761238.787544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761239.427619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761239.811596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761240.451654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761240.835529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761241.475676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761241.603578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761241.603605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761241.603610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761241.603620:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 691 00000100:00080000:3.0:1789761241.603631:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761241.603639:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761241.603680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048719104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761241.603724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761241.603727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761241.603729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761241.603732:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 691 00000100:00080000:3.0:1789761241.603735:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761241.603738:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761241.603747:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761241.603757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048719232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761241.603767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761241.603770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761241.603771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761241.603776:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 691 00000100:00100000:2.0:1789761241.603779:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048719104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761241.603779:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761241.603783:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761241.603790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761241.603801:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048719360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761241.603809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761241.603812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761241.603813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761241.603815:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 691 00000100:00080000:3.0:1789761241.603824:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761241.603828:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761241.603845:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761241.603873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048719488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761241.603881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761241.603883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761241.603885:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761241.603887:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 691 00000100:00080000:3.0:1789761241.603890:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761241.603893:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761241.603897:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761241.603899:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761241.603907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048719616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761241.603979:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761241/real 1789761241] req@ffff9811e1a22d80 x1876700048719104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761241.604015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048719104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761241.604024:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761241.604028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761241.604031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048719104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761241.604053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048719232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761241.604074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761241.604084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761241/real 1789761241] req@ffff9811e1a22680 x1876700048719232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761241.604093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048719232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761241.604099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761241.604102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761241.604104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048719232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761241.604108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761241.604119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761241.604141:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761241/real 1789761241] req@ffff9811e1a22300 x1876700048719360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761241.604156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048719360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761241.604165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761241.604169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761241.604189:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761241.604197:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761241.604218:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761241.604230:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761241/real 1789761241] req@ffff9811e1a21c00 x1876700048719488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761241.604270:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048719488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761241.604280:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761241.604284:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761241.604287:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761241.604294:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761241.604330:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761241.604339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761241/real 1789761241] req@ffff9811e1a22a00 x1876700048719616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761241.604348:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048719616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761241.604362:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761241.604364:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761241.604384:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048719616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761241.859594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761242.499705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761242.883518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761243.523659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761243.907566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761244.547645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761244.931532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761245.571596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761245.955561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761246.595621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761246.723627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761246.723639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761246.723643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761246.723654:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 696 00000100:00080000:0.0:1789761246.723666:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761246.723674:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761246.723723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048719744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761246.723786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761246.723790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761246.723803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761246.723808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 696 00000100:00080000:0.0:1789761246.723811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761246.723815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761246.723825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761246.723846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048719872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761246.723861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761246.723870:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048719744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761246.723871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761246.723878:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761246.723890:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 696 00000100:00080000:0.0:1789761246.723893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761246.723897:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761246.723902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761246.723917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048720000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761246.723924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761246.723927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761246.723929:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761246.723932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 696 00000100:00080000:0.0:1789761246.723946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761246.723951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761246.723958:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761246.723971:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761246.723977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048720128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761246.723993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761246.723996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761246.723997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761246.724001:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 696 00000100:00080000:0.0:1789761246.724005:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761246.724009:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761246.724013:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761246.724022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761246/real 1789761246] req@ffff9811c2e2aa00 x1876700048719744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761246.724028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048720256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761246.724053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048719744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761246.724065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761246.724068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761246.724071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048719744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761246.724098:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048719872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761246.724117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761246.724126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761246/real 1789761246] req@ffff9811c2e2b100 x1876700048719872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761246.724134:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048719872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761246.724139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761246.724142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761246.724144:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048719872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761246.724162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761246.724176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761246.724183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761246/real 1789761246] req@ffff9811c2e2bb80 x1876700048720000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761246.724191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048720000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761246.724196:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761246.724198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761246.724200:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761246.724203:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761246.724213:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761246.724219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761246/real 1789761246] req@ffff9811c2e29880 x1876700048720128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761246.724227:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048720128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761246.724232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761246.724233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761246.724235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761246.724239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761246.724248:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761246.724253:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761246/real 1789761246] req@ffff9811c2e2a680 x1876700048720256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761246.724265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048720256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761246.724270:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761246.724272:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761246.724274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761246.979545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761247.619612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761248.003591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761248.643709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761249.027574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761249.667667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761250.051576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761250.691617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761251.075678:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761251.715662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761251.843552:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761251.843561:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761251.843566:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761251.843575:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 701 00000100:00080000:3.0:1789761251.843585:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761251.843593:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761251.843630:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048720384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761251.843658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761251.843675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761251.843677:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761251.843680:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 701 00000100:00080000:3.0:1789761251.843684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761251.843687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761251.843696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761251.843708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048720512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761251.843716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761251.843719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761251.843720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761251.843723:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 701 00000100:00080000:3.0:1789761251.843725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761251.843727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761251.843731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761251.843739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048720640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761251.843747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761251.843750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761251.843752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761251.843766:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048720384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761251.843768:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 701 00000100:00080000:3.0:1789761251.843775:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761251.843783:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761251.843792:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761251.843803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048720768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761251.843810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761251.843812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761251.843813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761251.843816:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 701 00000100:00080000:3.0:1789761251.843819:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761251.843822:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761251.843825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761251.843832:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048720896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761251.843868:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761251.843925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761251/real 1789761251] req@ffff9811e1a21180 x1876700048720384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761251.843949:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048720384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761251.843962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761251.843967:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761251.843971:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048720384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761251.843980:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048720512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761251.844004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761251.844017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761251/real 1789761251] req@ffff9811e1a21880 x1876700048720512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761251.844030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048720512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761251.844060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761251.844064:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761251.844068:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048720512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761251.844098:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761251.844123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761251.844135:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761251/real 1789761251] req@ffff9811e1a20700 x1876700048720640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761251.844159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048720640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761251.844167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761251.844171:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761251.844174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761251.844181:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761251.844204:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761251.844216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761251/real 1789761251] req@ffff9811e1a23100 x1876700048720768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761251.844229:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048720768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761251.844246:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761251.844249:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761251.844259:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761251.844267:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761251.844290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761251.844302:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761251/real 1789761251] req@ffff9811e1a23480 x1876700048720896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761251.844315:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048720896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761251.844323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761251.844327:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761251.844330:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048720896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761252.099608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761252.739668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761253.123536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761253.763650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761254.147639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761254.787603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761255.171584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761255.811686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761256.195637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761256.835620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761256.963621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761256.963629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761256.963632:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761256.963640:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 706 00000100:00080000:0.0:1789761256.963649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761256.963655:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761256.963686:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048721024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761256.963710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761256.963713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761256.963714:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761256.963731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 706 00000100:00080000:0.0:1789761256.963734:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761256.963737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761256.963745:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761256.963755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048721152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761256.963764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761256.963768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761256.963769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761256.963773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 706 00000100:00080000:0.0:1789761256.963775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761256.963801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761256.963801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048721024:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761256.963814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761256.963835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048721280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761256.963845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761256.963848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761256.963850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761256.963853:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 706 00000100:00080000:0.0:1789761256.963856:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761256.963860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761256.963866:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761256.963875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048721408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761256.963893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761256.963893:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761256.963903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761256.963906:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761256.963911:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 706 00000100:00080000:0.0:1789761256.963915:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761256.963919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761256.963924:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761256.963935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048721536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761256.963961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761256/real 1789761256] req@ffff9811c2e28700 x1876700048721024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761256.963986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048721024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761256.964000:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761256.964005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761256.964010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048721024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761256.964018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048721152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761256.964040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761256.964064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761256/real 1789761256] req@ffff9811c2e2ad80 x1876700048721152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761256.964073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048721152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761256.964079:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761256.964083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761256.964087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048721152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761256.964120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761256.964142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761256.964152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761256/real 1789761256] req@ffff9811c2e29c00 x1876700048721280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761256.964160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048721280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761256.964165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761256.964168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761256.964170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761256.964174:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761256.964186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761256.964191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761256/real 1789761256] req@ffff9811c2e2a680 x1876700048721408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761256.964201:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048721408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761256.964215:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761256.964219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761256.964223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761256.964229:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761256.964252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761256.964261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761256/real 1789761256] req@ffff9811c2e29880 x1876700048721536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761256.964269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048721536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761256.964274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761256.964276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761256.964278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761257.219610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761257.859634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761258.243622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761258.883628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761259.267649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761259.907623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761260.291601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761260.931627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761261.315606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761261.955611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761262.083536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761262.083546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761262.083550:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761262.083561:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 711 00000100:00080000:0.0:1789761262.083571:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761262.083580:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761262.083617:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048721664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761262.083647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761262.083651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761262.083653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761262.083657:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 711 00000100:00080000:0.0:1789761262.083662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761262.083666:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761262.083690:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761262.083705:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048721792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761262.083714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048721664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761262.083722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761262.083728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761262.083743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761262.083748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 711 00000100:00080000:0.0:1789761262.083752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761262.083755:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761262.083760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761262.083770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048721920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761262.083777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761262.083779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761262.083780:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761262.083783:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 711 00000100:00080000:0.0:1789761262.083785:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761262.083787:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761262.083790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761262.083797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048722048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761262.083802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761262.083804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761262.083806:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761262.083808:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 711 00000100:00080000:0.0:1789761262.083811:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761262.083814:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761262.083818:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761262.083835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048722176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761262.083835:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048721792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761262.083882:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761262.083912:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761262.083943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761262.339552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761262.979625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761263.363599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761264.003638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761264.387613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761265.027726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761265.155828:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789761265.155885:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761265.155899:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761265.155935:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761265.155948:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761265.155961:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761265.155976:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761265.155998:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761262/real 1789761265] req@ffff9811c2e2b100 x1876700048721792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761265.156056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048721792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761265.156070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761265.156077:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761265.156082:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048721792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761265.156091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761262/real 1789761265] req@ffff9811c2e2aa00 x1876700048721920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761265.156104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048721920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761265.156112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761265.156116:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761265.156119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048721920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761265.156127:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761262/real 1789761265] req@ffff9811c2e29180 x1876700048722048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761265.156140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048722048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761265.156149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761265.156153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761265.156156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761265.156164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761262/real 1789761265] req@ffff9811c2e2bb80 x1876700048721664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761265.156197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048721664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761265.156212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761265.156219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761265.156226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048721664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761265.156285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761262/real 1789761265] req@ffff9811c2e28e00 x1876700048722176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761265.156300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048722176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761265.156309:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761265.156313:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761265.156317:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761265.411572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761266.051645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761266.435584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761267.075613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761267.203594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761267.203602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761267.203605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761267.203614:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 716 00000100:00080000:3.0:1789761267.203624:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761267.203630:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761267.203674:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048722304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761267.203699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761267.203703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761267.203705:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761267.203709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 716 00000100:00080000:3.0:1789761267.203730:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761267.203735:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761267.203746:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761267.203755:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048722304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761267.203765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048722432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761267.203780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761267.203784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761267.203786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761267.203792:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 716 00000100:00080000:3.0:1789761267.203796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761267.203800:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761267.203806:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761267.203820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048722560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761267.203825:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761267.203837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761267.203841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761267.203843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761267.203848:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 716 00000100:00080000:3.0:1789761267.203852:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761267.203856:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761267.203863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761267.203869:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761267/real 1789761267] req@ffff9811e1a20000 x1876700048722304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761267.203878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048722688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761267.203898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048722304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761267.203914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761267.203920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761267.203922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761267.203923:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761267.203933:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761267.203940:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048722304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761267.203949:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 716 00000100:00080000:3.0:1789761267.203959:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761267.203966:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761267.203973:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761267.203982:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048722432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761267.203990:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048722816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761267.204015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761267.204030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761267/real 1789761267] req@ffff9811e1a23800 x1876700048722432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761267.204044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048722432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761267.204053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761267.204057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761267.204061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048722432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761267.204067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761267.204084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761267.204093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761267/real 1789761267] req@ffff9811e1a21f80 x1876700048722560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761267.204105:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048722560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761267.204114:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761267.204117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761267.204121:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761267.204127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761267.204143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761267.204153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761267/real 1789761267] req@ffff9811e1a20a80 x1876700048722688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761267.204175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048722688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761267.204184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761267.204187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761267.204191:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761267.204209:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761267.204233:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761267.204244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761267/real 1789761267] req@ffff9811e1a20e00 x1876700048722816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761267.204257:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048722816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761267.204265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761267.204268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761267.204272:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048722816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761267.459559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761268.099690:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761268.483557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761269.123676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761269.507541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761270.147615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761270.531525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761271.171666:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761271.555579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761272.195593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761272.323583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761272.323609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761272.323614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761272.323625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 721 00000100:00080000:1.0:1789761272.323636:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761272.323645:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761272.323679:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048722944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761272.323709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761272.323714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761272.323717:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761272.323723:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 721 00000100:00080000:1.0:1789761272.323742:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761272.323748:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761272.323759:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761272.323771:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048722944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761272.323777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048723072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761272.323792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761272.323796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761272.323798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761272.323802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 721 00000100:00080000:1.0:1789761272.323806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761272.323825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761272.323830:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761272.323842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048723200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761272.323853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761272.323859:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761272.323860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761272.323865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761272.323871:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 721 00000100:00080000:1.0:1789761272.323899:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761272.323905:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761272.323911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761272.323913:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761272/real 1789761272] req@ffff9811c92c4380 x1876700048722944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761272.323927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048723328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761272.323932:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048722944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761272.323950:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761272.323958:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761272.323963:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048722944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761272.323964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761272.323969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761272.323972:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761272.323978:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 721 00000100:00080000:1.0:1789761272.323983:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761272.323985:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048723072:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761272.323991:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761272.324008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761272.324012:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761272.324020:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761272/real 1789761272] req@ffff9811c92c5500 x1876700048723072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761272.324030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048723456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761272.324032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048723072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761272.324062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761272.324067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761272.324070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048723072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761272.324076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761272.324089:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761272.324099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761272/real 1789761272] req@ffff9811c92c5880 x1876700048723200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761272.324107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048723200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761272.324112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761272.324114:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761272.324117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761272.324129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761272.324150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761272.324162:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761272/real 1789761272] req@ffff9811c92c4e00 x1876700048723328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761272.324172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048723328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761272.324178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761272.324180:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761272.324182:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761272.324195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761272.324208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761272.324220:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761272/real 1789761272] req@ffff9811c92c4000 x1876700048723456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761272.324232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048723456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761272.324238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761272.324240:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761272.324242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761272.579527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761273.219613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761273.603522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761274.243635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761274.627657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761275.267659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761275.651594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761276.291630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761276.675547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761277.315583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761277.443588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761277.443598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761277.443602:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761277.443623:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 727 00000100:00080000:0.0:1789761277.443633:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761277.443641:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761277.443681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048723584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761277.443717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761277.443720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761277.443721:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761277.443725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 727 00000100:00080000:0.0:1789761277.443728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761277.443732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761277.443741:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761277.443754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048723712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761277.443754:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048723584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761277.443769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761277.443772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761277.443773:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761277.443777:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 727 00000100:00080000:0.0:1789761277.443779:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761277.443782:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761277.443812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761277.443814:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761277.443825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048723840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761277.443834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761277.443836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761277.443838:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761277.443841:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 727 00000100:00080000:0.0:1789761277.443845:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761277.443849:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761277.443852:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761277/real 1789761277] req@ffff9811c2e28380 x1876700048723584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761277.443861:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761277.443872:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048723584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761277.443874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048723968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761277.443888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761277.443890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761277.443892:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761277.443899:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 727 00000100:00080000:0.0:1789761277.443904:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761277.443907:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761277.443910:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761277.443918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048724096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761277.443935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761277.443944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761277.443950:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048723584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761277.443974:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048723712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761277.443994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761277.444004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761277/real 1789761277] req@ffff9811c2e28a80 x1876700048723712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761277.444014:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048723712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761277.444020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761277.444022:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761277.444024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048723712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761277.444028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761277.444040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761277.444046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761277/real 1789761277] req@ffff9811c2e29f80 x1876700048723840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761277.444053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048723840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761277.444058:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761277.444060:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761277.444062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761277.444072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761277.444092:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761277.444104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761277/real 1789761277] req@ffff9811c2e29500 x1876700048723968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761277.444126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048723968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761277.444136:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761277.444138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761277.444141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048723968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761277.444145:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761277.444167:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761277.444179:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761277/real 1789761277] req@ffff9811c2e2b480 x1876700048724096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761277.444192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048724096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761277.444201:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761277.444204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761277.444206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761277.699551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761278.340574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761278.723595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761279.363612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761279.747563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761280.387609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761280.771599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761281.411637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761281.795544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761282.435611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761282.563605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761282.563616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761282.563620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761282.563653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 732 00000100:00080000:2.0:1789761282.563664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761282.563671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761282.563724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048724224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761282.563750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761282.563753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761282.563754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761282.563757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 732 00000100:00080000:2.0:1789761282.563765:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761282.563768:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761282.563780:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761282.563790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048724352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761282.563809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048724224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761282.563811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761282.563818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761282.563821:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761282.563824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 732 00000100:00080000:2.0:1789761282.563827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761282.563829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761282.563832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761282.563841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048724480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761282.563858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761282.563861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761282.563862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761282.563865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 732 00000100:00080000:2.0:1789761282.563869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761282.563883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761282.563887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761282.563901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048724608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761282.563905:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761282.563917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761282.563922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761282.563924:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761282.563931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 732 00000100:00080000:2.0:1789761282.563935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761282.563951:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761282.563956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761282.563968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048724736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761282.563975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761282/real 1789761282] req@ffff9811d8639180 x1876700048724224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761282.563999:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048724224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761282.564023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761282.564028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761282.564033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048724224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761282.564061:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048724352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761282.564093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761282.564115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761282/real 1789761282] req@ffff9811d863bb80 x1876700048724352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761282.564133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048724352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761282.564143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761282.564147:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761282.564151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048724352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761282.564158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761282.564190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761282.564202:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761282/real 1789761282] req@ffff9811d863b100 x1876700048724480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761282.564216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048724480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761282.564225:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761282.564230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761282.564233:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761282.564240:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761282.564260:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761282.564270:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761282/real 1789761282] req@ffff9811d8639c00 x1876700048724608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761282.564283:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048724608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761282.564304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761282.564308:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761282.564311:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761282.564318:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761282.564336:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761282.564347:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761282/real 1789761282] req@ffff9811d863ad80 x1876700048724736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761282.564388:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048724736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761282.564398:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761282.564402:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761282.564406:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048724736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761282.819620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761283.459695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761283.843561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761284.483660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761284.867619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761285.507615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761285.891575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761286.531656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761286.915573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761287.555614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761287.683580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761287.683588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761287.683591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761287.683599:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 737 00000100:00080000:2.0:1789761287.683621:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761287.683627:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761287.683659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048724864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761287.683685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761287.683689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761287.683691:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761287.683696:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 737 00000100:00080000:2.0:1789761287.683701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761287.683706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761287.683715:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761287.683729:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048724864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761287.683729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048724992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761287.683753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761287.683755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761287.683756:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761287.683760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 737 00000100:00080000:2.0:1789761287.683763:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761287.683766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761287.683770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761287.683778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048725120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761287.683785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761287.683793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761287.683794:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761287.683797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 737 00000100:00080000:2.0:1789761287.683800:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761287.683803:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761287.683807:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761287.683820:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761287.683824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048725248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761287.683834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761287.683837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761287.683838:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761287.683841:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 737 00000100:00080000:2.0:1789761287.683844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761287.683846:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761287.683849:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761287.683856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048725376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761287.683888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761287/real 1789761287] req@ffff9811d8639880 x1876700048724864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761287.683909:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048724864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761287.683918:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761287.683922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761287.683926:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048724864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761287.683951:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048724992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761287.683973:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761287.683987:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761287/real 1789761287] req@ffff9811d8639500 x1876700048724992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761287.684016:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048724992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761287.684025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761287.684029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761287.684034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048724992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761287.684041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761287.684066:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761287.684079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761287/real 1789761287] req@ffff9811d8639f80 x1876700048725120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761287.684093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048725120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761287.684101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761287.684105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761287.684109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761287.684116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761287.684149:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761287.684162:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761287/real 1789761287] req@ffff9811d863a680 x1876700048725248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761287.684175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048725248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761287.684200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761287.684205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761287.684209:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761287.684216:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761287.684252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761287.684265:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761287/real 1789761287] req@ffff9811d8638a80 x1876700048725376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761287.684288:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048725376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761287.684297:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761287.684301:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761287.684305:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761287.939583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761288.579616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761288.963593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761289.603652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761289.987620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761290.627633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761291.011576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761291.651600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761292.035645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761292.675662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761292.803570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761292.803582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761292.803586:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761292.803609:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 742 00000100:00080000:0.0:1789761292.803622:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761292.803634:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761292.803700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048725504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761292.803729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761292.803732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761292.803734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761292.803737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 742 00000100:00080000:0.0:1789761292.803740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761292.803744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761292.803755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761292.803784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048725632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761292.803799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761292.803803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761292.803805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761292.803809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 742 00000100:00080000:0.0:1789761292.803816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761292.803816:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048725504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761292.803826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761292.803832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761292.803845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048725760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761292.803853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761292.803856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761292.803857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761292.803868:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 742 00000100:00080000:0.0:1789761292.803871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761292.803875:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761292.803880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761292.803887:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761292.803893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048725888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761292.803905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761292.803908:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761292.803909:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761292.803912:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 742 00000100:00080000:0.0:1789761292.803915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761292.803918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761292.803924:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761292.803932:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761292/real 1789761292] req@ffff9811c2e2b800 x1876700048725504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761292.803939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048726016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761292.803994:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048725504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761292.804026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761292.804029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761292.804032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048725504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761292.804054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048725632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761292.804075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761292.804084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761292/real 1789761292] req@ffff9811c2e28000 x1876700048725632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761292.804093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048725632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761292.804099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761292.804101:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761292.804103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048725632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761292.804107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761292.804119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761292.804125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761292/real 1789761292] req@ffff9811c2e2a300 x1876700048725760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761292.804132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048725760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761292.804137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761292.804140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761292.804142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761292.804151:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761292.804169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761292.804180:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761292/real 1789761292] req@ffff9811c2e2b480 x1876700048725888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761292.804194:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048725888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761292.804202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761292.804206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761292.804210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048725888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761292.804217:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761292.804240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761292.804250:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761292/real 1789761292] req@ffff9811c2e29500 x1876700048726016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761292.804258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048726016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761292.804263:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761292.804265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761292.804267:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761293.059662:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761293.699562:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761294.083612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761294.723611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761295.107592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761295.747589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761296.131632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761296.771640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761297.155606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761297.795598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761297.923583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761297.923594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761297.923619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761297.923629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 747 00000100:00080000:0.0:1789761297.923641:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761297.923649:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761297.923689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048726144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761297.923722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761297.923727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761297.923729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761297.923734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 747 00000100:00080000:0.0:1789761297.923739:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761297.923743:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761297.923755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761297.923770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048726272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761297.923783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761297.923788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761297.923790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761297.923794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 747 00000100:00080000:0.0:1789761297.923798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761297.923803:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048726144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761297.923804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761297.923814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761297.923826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048726400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761297.923845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761297.923848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761297.923850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761297.923853:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 747 00000100:00080000:0.0:1789761297.923856:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761297.923859:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761297.923863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761297.923872:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048726528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761297.923879:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761297.923885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761297.923889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761297.923890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761297.923893:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 747 00000100:00080000:0.0:1789761297.923895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761297.923897:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761297.923900:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761297.923907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048726656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761297.923938:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761297/real 1789761297] req@ffff9811c2e29f80 x1876700048726144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761297.923970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048726144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761297.923989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761297.923993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761297.923997:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048726144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761297.924005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048726272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761297.924032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761297.924047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761297/real 1789761297] req@ffff9811c2e28a80 x1876700048726272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761297.924061:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048726272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761297.924069:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761297.924073:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761297.924076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048726272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761297.924106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761297.924125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761297.924134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761297/real 1789761297] req@ffff9811c2e28380 x1876700048726400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761297.924151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048726400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761297.924167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761297.924170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761297.924172:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761297.924176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761297.924193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761297.924200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761297/real 1789761297] req@ffff9811c2e28e00 x1876700048726528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761297.924208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048726528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761297.924213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761297.924215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761297.924217:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761297.924221:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761297.924230:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761297.924236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761297/real 1789761297] req@ffff9811c2e2bb80 x1876700048726656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761297.924243:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048726656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761297.924248:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761297.924249:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761297.924256:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048726656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761298.179655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761298.819621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761299.203637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761299.843612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761300.227626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761300.867609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761301.251527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761301.891582:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761302.275597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761302.915600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761303.043597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761303.043609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761303.043613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761303.043624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 752 00000100:00080000:0.0:1789761303.043636:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761303.043645:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761303.043683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048726784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761303.043710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761303.043714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761303.043716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761303.043720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 752 00000100:00080000:0.0:1789761303.043725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761303.043730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761303.043740:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761303.043742:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048726784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761303.043759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048726912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761303.043774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761303.043778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761303.043780:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761303.043784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 752 00000100:00080000:0.0:1789761303.043792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761303.043797:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789761303.043801:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761303.043819:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761303.043837:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048727040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761303.043839:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761303/real 1789761303] req@ffff9811c2e29180 x1876700048726784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761303.043855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761303.043862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761303.043867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761303.043874:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 752 00000100:00100000:2.0:1789761303.043880:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048726784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761303.043881:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761303.043890:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761303.043894:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761303.043898:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761303.043901:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761303.043906:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048726784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761303.043918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048727168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761303.043932:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048726912:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789761303.043933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761303.043940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761303.043944:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761303.043948:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 752 00000100:00080000:0.0:1789761303.043953:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761303.043958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761303.043967:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761303.043970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761303.043988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048727296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761303.043991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761303/real 1789761303] req@ffff9811c2e2aa00 x1876700048726912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761303.044027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048726912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761303.044038:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761303.044042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761303.044047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048726912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761303.044055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761303.044079:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761303.044095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761303/real 1789761303] req@ffff9811c2e2b100 x1876700048727040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761303.044104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048727040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761303.044110:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761303.044112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761303.044114:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761303.044126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761303.044139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761303.044146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761303/real 1789761303] req@ffff9811c2e29880 x1876700048727168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761303.044154:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048727168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761303.044159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761303.044161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761303.044163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761303.044167:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761303.044176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761303.044181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761303/real 1789761303] req@ffff9811c2e2a680 x1876700048727296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761303.044188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048727296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761303.044198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761303.044202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761303.044206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761303.299536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761303.939585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761304.323598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761304.963603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761305.347574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761305.987695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761306.371644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761307.011644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761307.395597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761308.035610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761308.163570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761308.163579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761308.163586:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761308.163595:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 757 00000100:00080000:1.0:1789761308.163604:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761308.163611:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761308.163646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048727424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761308.163676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761308.163680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761308.163684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761308.163689:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 757 00000100:00080000:1.0:1789761308.163694:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761308.163699:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761308.163707:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048727424:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789761308.163711:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761308.163727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048727552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761308.163753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761308.163757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761308.163768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761308.163773:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761308.163778:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 757 00000100:00080000:1.0:1789761308.163786:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761308.163792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761308.163799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761308.163814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048727680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761308.163825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761308.163826:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761308/real 1789761308] req@ffff9811c92c5f80 x1876700048727424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761308.163835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761308.163839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761308.163846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 757 00000100:00080000:1.0:1789761308.163850:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761308.163854:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048727424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761308.163859:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761308.163868:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761308.163875:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761308.163881:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761308.163884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048727808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761308.163889:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048727424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761308.163906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761308.163910:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761308.163912:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761308.163918:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 757 00000100:00080000:1.0:1789761308.163930:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761308.163933:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048727552:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761308.163938:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761308.163947:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761308.163963:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048727936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761308.163971:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761308.163990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761308/real 1789761308] req@ffff9811c92c7480 x1876700048727552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761308.164000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048727552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761308.164007:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761308.164010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761308.164013:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048727552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761308.164024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761308.164039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761308.164047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761308/real 1789761308] req@ffff9811c92c4a80 x1876700048727680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761308.164055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048727680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761308.164060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761308.164062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761308.164064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761308.164068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761308.164078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761308.164083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761308/real 1789761308] req@ffff9811c92c7800 x1876700048727808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761308.164095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048727808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761308.164103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761308.164106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761308.164110:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761308.164130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761308.164153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761308.164166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761308/real 1789761308] req@ffff9811c92c7100 x1876700048727936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761308.164188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048727936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761308.164199:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761308.164203:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761308.164207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048727936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761308.419572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761309.059580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761309.443577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761310.083665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761310.467630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761311.107589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761311.491599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761312.131662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761312.515642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761313.155572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761313.283603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761313.283614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761313.283619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761313.283630:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 762 00000100:00080000:1.0:1789761313.283641:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761313.283649:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761313.283690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048728064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761313.283717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761313.283720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761313.283722:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761313.283725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 762 00000100:00080000:1.0:1789761313.283727:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761313.283730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761313.283739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761313.283748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048728192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761313.283759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761313.283771:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048728064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761313.283780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761313.283782:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761313.283786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 762 00000100:00080000:1.0:1789761313.283789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761313.283792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761313.283797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761313.283806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048728320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761313.283814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761313.283817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761313.283818:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761313.283821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 762 00000100:00080000:1.0:1789761313.283832:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761313.283834:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761313.283837:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761313.283844:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761313.283849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048728448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761313.283857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761313.283860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761313.283862:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761313.283867:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 762 00000100:00080000:1.0:1789761313.283872:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761313.283877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761313.283890:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761313.283893:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761313/real 1789761313] req@ffff9811c92c6a00 x1876700048728064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761313.283907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048728576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761313.283919:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048728064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761313.283933:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761313.283939:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761313.283945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048728064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761313.283973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048728192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761313.284000:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761313.284011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761313/real 1789761313] req@ffff9811c92c4700 x1876700048728192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761313.284020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048728192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761313.284026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761313.284028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761313.284031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048728192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761313.284034:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761313.284045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761313.284051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761313/real 1789761313] req@ffff9811c92c6300 x1876700048728320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761313.284059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048728320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761313.284065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761313.284066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761313.284068:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761313.284071:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761313.284082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761313.284088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761313/real 1789761313] req@ffff9811c92c7100 x1876700048728448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761313.284112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048728448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761313.284118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761313.284120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761313.284122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761313.284126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761313.284140:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761313.284147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761313/real 1789761313] req@ffff9811c92c7800 x1876700048728576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761313.284155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048728576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761313.284160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761313.284162:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761313.284163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761313.539603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761314.179643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761314.563574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761315.203574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761315.587527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761316.227641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761316.611547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761317.251602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761317.635581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761318.275654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761318.403570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761318.403585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761318.403589:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761318.403601:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 767 00000100:00080000:1.0:1789761318.403615:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761318.403624:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761318.403673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048728704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761318.403701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761318.403704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761318.403705:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761318.403709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 767 00000100:00080000:1.0:1789761318.403712:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761318.403715:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761318.403724:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761318.403742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048728832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761318.403753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761318.403754:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048728704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761318.403756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761318.403760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761318.403764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 767 00000100:00080000:1.0:1789761318.403782:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761318.403790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761318.403794:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761318.403804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048728960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761318.403811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761318.403814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761318.403815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761318.403817:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 767 00000100:00080000:1.0:1789761318.403820:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761318.403822:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761318.403825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761318.403837:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048729088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761318.403844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761318.403846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761318.403848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761318.403850:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 767 00000100:00080000:1.0:1789761318.403853:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761318.403855:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761318.403857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761318.403866:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761318.403868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048729216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761318.403926:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761318/real 1789761318] req@ffff9811c92c4a80 x1876700048728704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761318.403945:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048728704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761318.403953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761318.403957:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761318.403960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048728704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761318.403984:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048728832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761318.404002:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761318.404012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761318/real 1789761318] req@ffff9811c92c7480 x1876700048728832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761318.404020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048728832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761318.404026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761318.404028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761318.404031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048728832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761318.404034:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761318.404045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761318.404051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761318/real 1789761318] req@ffff9811c92c5f80 x1876700048728960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761318.404070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048728960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761318.404086:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761318.404091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761318.404093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048728960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761318.404097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761318.404115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761318.404126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761318/real 1789761318] req@ffff9811c92c4000 x1876700048729088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761318.404135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048729088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761318.404141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761318.404143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761318.404145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761318.404148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761318.404158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761318.404164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761318/real 1789761318] req@ffff9811c92c4e00 x1876700048729216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761318.404171:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048729216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761318.404186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761318.404189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761318.404192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761318.659564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761319.299658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761319.683588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761320.323648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761320.707583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761321.347628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761321.731554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761322.371609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761322.755542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761323.395652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761323.523585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761323.523598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761323.523604:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761323.523617:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 773 00000100:00080000:1.0:1789761323.523631:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761323.523641:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761323.523689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048729344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761323.523722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761323.523727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761323.523729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761323.523733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 773 00000100:00080000:1.0:1789761323.523738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761323.523742:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761323.523754:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761323.523770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048729472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761323.523786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761323.523791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761323.523792:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048729344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761323.523795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761323.523811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 773 00000100:00080000:1.0:1789761323.523816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761323.523822:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761323.523847:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761323.523862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048729600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761323.523874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761323.523882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789761323.523886:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761323.523887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761323.523898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 773 00000100:00080000:1.0:1789761323.523906:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761323.523912:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761323.523919:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761323.523933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048729728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761323.523944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761323.523953:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761323/real 1789761323] req@ffff9811c92c5880 x1876700048729344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761323.523953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761323.523960:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761323.523980:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 773 00000100:00080000:1.0:1789761323.523989:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761323.523997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761323.523999:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048729344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761323.524006:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761323.524028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761323.524036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048729856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761323.524037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761323.524051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048729344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761323.524091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048729472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761323.524119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761323.524143:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761323/real 1789761323] req@ffff9811c92c5500 x1876700048729472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761323.524153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048729472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761323.524158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761323.524161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761323.524163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048729472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761323.524168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761323.524181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761323.524188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761323/real 1789761323] req@ffff9811c92c4380 x1876700048729600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761323.524195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048729600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761323.524200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761323.524202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761323.524203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761323.524207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761323.524235:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761323.524244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761323/real 1789761323] req@ffff9811c92c6680 x1876700048729728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761323.524252:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048729728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761323.524262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761323.524265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761323.524268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761323.524272:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761323.524287:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761323.524294:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761323/real 1789761323] req@ffff9811c92c5c00 x1876700048729856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761323.524301:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048729856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761323.524306:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761323.524308:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761323.524310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048729856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761323.779642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761324.419614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761324.803570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761325.443672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761325.827581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761326.467610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761326.851602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761327.491612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761327.875553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761328.515608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761328.643580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761328.643588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761328.643605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761328.643619:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 778 00000100:00080000:0.0:1789761328.643627:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761328.643633:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761328.643660:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048729984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761328.643683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761328.643685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761328.643687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761328.643690:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 778 00000100:00080000:0.0:1789761328.643692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761328.643694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761328.643701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761328.643709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048730112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761328.643717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761328.643720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761328.643721:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761328.643724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 778 00000100:00080000:0.0:1789761328.643726:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761328.643731:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048729984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761328.643733:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761328.643742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761328.643755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048730240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761328.643780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761328.643783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761328.643786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761328.643790:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 778 00000100:00080000:0.0:1789761328.643804:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761328.643810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761328.643815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761328.643828:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048730112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761328.643830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048730368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761328.643840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761328.643843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761328.643844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761328.643848:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 778 00000100:00080000:0.0:1789761328.643854:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761328.643860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761328.643865:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761328.643867:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761328.643883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048730496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761328.643888:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761328.643917:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761328.899582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761329.539604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761329.923592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761330.563596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761330.947582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761331.587635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761331.715750:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789761331.715821:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761331.715840:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761331.715879:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761331.715886:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761331.715895:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761331.715926:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761331.715953:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761328/real 1789761331] req@ffff9811c2e28700 x1876700048730240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761331.715987:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048730240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761331.716003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761331.716009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761331.716015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761331.716024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761328/real 1789761331] req@ffff9811c2e2a680 x1876700048730368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761331.716040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048730368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761331.716064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761331.716068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761331.716072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761331.716080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761328/real 1789761331] req@ffff9811c2e29c00 x1876700048729984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761331.716095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048729984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761331.716103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761331.716107:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761331.716110:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048729984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761331.716118:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761328/real 1789761331] req@ffff9811c2e2ad80 x1876700048730112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761331.716132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048730112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761331.716139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761331.716143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761331.716147:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048730112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761331.716197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761328/real 1789761331] req@ffff9811c2e29880 x1876700048730496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761331.716212:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048730496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761331.716228:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761331.716233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761331.716237:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761331.971575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761332.611604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761332.995586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761333.635651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761333.763575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761333.763588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761333.763613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761333.763626:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 783 00000100:00080000:1.0:1789761333.763639:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761333.763649:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761333.763693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048730624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761333.763730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761333.763735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761333.763737:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761333.763742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 783 00000100:00080000:1.0:1789761333.763747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761333.763752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761333.763766:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761333.763782:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048730624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761333.763785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048730752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761333.763804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761333.763809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761333.763811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761333.763816:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 783 00000100:00080000:1.0:1789761333.763826:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761333.763831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761333.763838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761333.763850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048730880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761333.763876:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761333.763883:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761333.763888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761333.763891:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761333.763897:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 783 00000100:00080000:1.0:1789761333.763910:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761333.763915:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761333.763921:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761333.763941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048731008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761333.763948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761333/real 1789761333] req@ffff9811c92c7b80 x1876700048730624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761333.763952:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761333.763960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761333.763964:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761333.763972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 783 00000100:00080000:1.0:1789761333.763975:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761333.763979:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761333.763984:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761333.763990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048730624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761333.763998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048731136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761333.764012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761333.764019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761333.764023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048730624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761333.764054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048730752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761333.764084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761333.764100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761333/real 1789761333] req@ffff9811c92c5180 x1876700048730752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761333.764113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048730752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761333.764122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761333.764132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761333.764136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048730752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761333.764170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761333.764200:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761333.764215:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761333/real 1789761333] req@ffff9811c92c6d80 x1876700048730880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761333.764235:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048730880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761333.764244:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761333.764248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761333.764251:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048730880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761333.764258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761333.764284:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761333.764297:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761333/real 1789761333] req@ffff9811c92c5c00 x1876700048731008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761333.764310:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048731008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761333.764318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761333.764322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761333.764325:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761333.764332:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761333.764353:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761333.764365:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761333/real 1789761333] req@ffff9811c92c6680 x1876700048731136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761333.764388:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048731136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761333.764396:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761333.764399:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761333.764403:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761334.019554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761334.659596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761335.043558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761335.683658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761336.067580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761336.707594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761337.091524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761337.731624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761338.115523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761338.755590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761338.883605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761338.883613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761338.883617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761338.883625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 788 00000100:00080000:2.0:1789761338.883634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761338.883640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761338.883671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048731264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761338.883697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761338.883700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761338.883702:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761338.883704:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 788 00000100:00080000:2.0:1789761338.883706:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761338.883708:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761338.883716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761338.883724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048731392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761338.883748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048731264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761338.883755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761338.883758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761338.883760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761338.883763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 788 00000100:00080000:2.0:1789761338.883767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761338.883769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761338.883773:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761338.883783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048731520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761338.883790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761338.883792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761338.883794:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761338.883796:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 788 00000100:00080000:2.0:1789761338.883798:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761338.883800:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761338.883804:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761338.883805:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761338.883814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048731648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761338.883823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761338.883825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761338.883826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761338.883828:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 788 00000100:00080000:2.0:1789761338.883837:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761338.883840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761338.883844:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761338.883845:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761338/real 1789761338] req@ffff9811d863aa00 x1876700048731264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761338.883857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048731776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761338.883864:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048731264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761338.883876:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761338.883879:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761338.883882:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048731264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761338.883902:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048731392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761338.883916:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761338.883925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761338/real 1789761338] req@ffff9811d8638000 x1876700048731392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761338.883944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048731392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761338.883950:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761338.883953:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761338.883955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048731392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761338.883958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761338.883971:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761338.883978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761338/real 1789761338] req@ffff9811d863b800 x1876700048731520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761338.883985:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048731520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761338.883990:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761338.883992:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761338.883994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761338.883998:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761338.884008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761338.884013:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761338/real 1789761338] req@ffff9811d863b480 x1876700048731648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761338.884020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048731648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761338.884033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761338.884035:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761338.884038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761338.884041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761338.884062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761338.884070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761338/real 1789761338] req@ffff9811d8638380 x1876700048731776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761338.884077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048731776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761338.884082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761338.884084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761338.884086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048731776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761339.139516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761339.779584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761340.163519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761340.803634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761341.187541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761341.827644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761342.211545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761342.851633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761343.235546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761343.875626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761344.003598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761344.003607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761344.003610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761344.003618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 793 00000100:00080000:2.0:1789761344.003628:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761344.003634:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761344.003668:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048731904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761344.003694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761344.003699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761344.003702:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761344.003709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 793 00000100:00080000:2.0:1789761344.003715:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761344.003718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761344.003726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761344.003742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048732032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761344.003754:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761344.003755:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048731904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761344.003773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761344.003776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761344.003781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 793 00000100:00080000:2.0:1789761344.003785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761344.003788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761344.003793:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761344.003805:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048732160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761344.003814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761344.003816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761344.003818:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761344.003821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 793 00000100:00080000:2.0:1789761344.003823:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761344.003828:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761344.003834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761344.003846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048732288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761344.003855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761344.003865:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761344.003866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761344.003870:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761344.003878:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 793 00000100:00080000:2.0:1789761344.003885:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761344.003890:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761344.003900:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761344.003910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048732416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761344.003942:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761344/real 1789761344] req@ffff9811d8638e00 x1876700048731904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761344.003977:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048731904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761344.003987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761344.003991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761344.003995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048731904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761344.004019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048732032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761344.004038:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761344.004048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761344/real 1789761344] req@ffff9811d8638700 x1876700048732032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761344.004057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048732032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761344.004075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761344.004080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761344.004083:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048732032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761344.004091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761344.004123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761344.004137:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761344/real 1789761344] req@ffff9811d863a300 x1876700048732160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761344.004148:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048732160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761344.004163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761344.004166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761344.004169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761344.004174:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761344.004199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761344.004213:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761344/real 1789761344] req@ffff9811d8638380 x1876700048732288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761344.004228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048732288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761344.004239:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761344.004244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761344.004250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761344.004274:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761344.004298:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761344.004321:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761344/real 1789761344] req@ffff9811d863b480 x1876700048732416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761344.004347:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048732416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761344.004356:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761344.004360:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761344.004373:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761344.259552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761344.899589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761345.283585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761345.923571:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761346.307584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761346.947589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761347.331578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761347.971596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761348.355571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761348.995596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761349.123563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761349.123574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761349.123579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761349.123589:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 798 00000100:00080000:2.0:1789761349.123602:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761349.123609:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761349.123651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048732544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761349.123676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761349.123679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761349.123681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761349.123684:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 798 00000100:00080000:2.0:1789761349.123691:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761349.123694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761349.123702:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761349.123713:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048732672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761349.123720:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048732544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761349.123724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761349.123728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761349.123730:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761349.123733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 798 00000100:00080000:2.0:1789761349.123752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761349.123755:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761349.123760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761349.123769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048732800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761349.123777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761349.123780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761349.123781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761349.123784:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 798 00000100:00080000:2.0:1789761349.123786:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761349.123789:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761349.123792:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761349.123797:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761349.123803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048732928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761349.123810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761349.123814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761349.123817:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761349.123822:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 798 00000100:00080000:2.0:1789761349.123826:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761349.123830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761349.123836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761349.123846:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761349/real 1789761349] req@ffff9811d863b800 x1876700048732544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761349.123851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048733056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761349.123871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048732544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761349.123893:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761349.123896:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761349.123900:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048732544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761349.123923:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048732672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761349.123939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761349.123948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761349/real 1789761349] req@ffff9811d8638000 x1876700048732672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761349.123957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048732672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761349.123963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761349.123965:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761349.123967:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048732672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761349.123971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761349.123981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761349.123997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761349/real 1789761349] req@ffff9811d863aa00 x1876700048732800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761349.124005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048732800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761349.124010:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761349.124012:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761349.124015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761349.124018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761349.124047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761349.124055:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761349/real 1789761349] req@ffff9811d8638a80 x1876700048732928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761349.124064:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048732928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761349.124075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761349.124077:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761349.124079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048732928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761349.124084:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761349.124096:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761349.124102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761349/real 1789761349] req@ffff9811d863a680 x1876700048733056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761349.124110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048733056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761349.124121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761349.124123:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761349.124125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761349.379621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761350.019583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761350.403566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761351.043630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761351.427588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761352.067589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761352.451571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761353.091656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761353.475586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761354.115649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761354.243565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761354.243573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761354.243577:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761354.243586:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 803 00000100:00080000:0.0:1789761354.243595:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761354.243601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761354.243631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048733184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761354.243659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761354.243663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761354.243665:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761354.243683:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 803 00000100:00080000:0.0:1789761354.243688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761354.243691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761354.243700:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761354.243702:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048733184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761354.243713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048733312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761354.243723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761354.243726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761354.243727:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761354.243730:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 803 00000100:00080000:0.0:1789761354.243732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761354.243734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761354.243738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761354.243746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048733440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761354.243752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761354.243755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761354.243767:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761354.243773:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761354.243777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 803 00000100:00080000:0.0:1789761354.243780:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761354.243783:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761354.243788:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761354.243801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048733568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761354.243819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761354.243823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761354.243825:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761354.243828:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761354/real 1789761354] req@ffff9811c2e2b100 x1876700048733184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761354.243833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 803 00000100:00080000:0.0:1789761354.243844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761354.243851:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048733184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761354.243852:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761354.243861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761354.243868:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761354.243873:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761354.243877:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048733184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761354.243882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048733696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761354.243901:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048733312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761354.243934:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761354.243949:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761354/real 1789761354] req@ffff9811c2e2aa00 x1876700048733312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761354.243963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048733312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761354.243973:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761354.243976:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761354.243979:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048733312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761354.244003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761354.244021:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761354.244033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761354/real 1789761354] req@ffff9811c2e29180 x1876700048733440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761354.244046:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048733440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761354.244055:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761354.244059:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761354.244063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761354.244069:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761354.244087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761354.244095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761354/real 1789761354] req@ffff9811c2e2bb80 x1876700048733568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761354.244102:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048733568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761354.244108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761354.244110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761354.244112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761354.244115:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761354.244125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761354.244130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761354/real 1789761354] req@ffff9811c2e28e00 x1876700048733696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761354.244144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048733696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761354.244149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761354.244151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761354.244153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048733696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761354.499609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761355.139607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761355.523538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761356.163613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761356.547688:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761357.187612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761357.571600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761358.211651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761358.595591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761359.235655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761359.363594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761359.363605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761359.363609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761359.363631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 808 00000100:00080000:0.0:1789761359.363644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761359.363652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761359.363696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048733824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761359.363723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761359.363744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761359.363745:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761359.363750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 808 00000100:00080000:0.0:1789761359.363757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761359.363762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761359.363764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048733824:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761359.363772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761359.363786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048733952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761359.363798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761359.363801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761359.363803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761359.363807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 808 00000100:00080000:0.0:1789761359.363827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761359.363831:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761359.363834:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761359.363836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761359.363852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048734080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761359.363861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761359.363864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761359.363866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761359.363872:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 808 00000100:00080000:0.0:1789761359.363877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761359.363882:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761359/real 1789761359] req@ffff9811c2e28380 x1876700048733824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761359.363885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761359.363900:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048733824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761359.363902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761359.363925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761359.363932:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761359.363933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048734208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761359.363943:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048733824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761359.363970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761359.363976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761359.363979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761359.363985:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 808 00000100:00080000:0.0:1789761359.363992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761359.363993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048733952:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789761359.364008:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761359.364015:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761359.364023:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761359.364028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048734336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761359.364038:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761359/real 1789761359] req@ffff9811c2e28a80 x1876700048733952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761359.364058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048733952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761359.364082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761359.364085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761359.364088:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048733952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761359.364093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761359.364109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761359.364116:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761359/real 1789761359] req@ffff9811c2e29f80 x1876700048734080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761359.364124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048734080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761359.364129:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761359.364131:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761359.364133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761359.364136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761359.364147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761359.364153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761359/real 1789761359] req@ffff9811c2e29500 x1876700048734208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761359.364160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048734208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761359.364165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761359.364167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761359.364174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761359.364187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761359.364200:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761359.364206:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761359/real 1789761359] req@ffff9811c2e2b480 x1876700048734336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761359.364219:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048734336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761359.364227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761359.364231:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761359.364234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761359.619564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761360.259614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761360.643584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761361.283589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761361.667620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761362.307643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761362.691584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761363.331655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761363.715649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761364.355642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761364.483587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761364.483599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761364.483605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761364.483617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 814 00000100:00080000:0.0:1789761364.483630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761364.483638:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761364.483679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048734464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761364.483723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761364.483726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761364.483728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761364.483731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 814 00000100:00080000:0.0:1789761364.483737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761364.483741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761364.483752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761364.483768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048734592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761364.483768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048734464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761364.483798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761364.483803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761364.483805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761364.483810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 814 00000100:00080000:0.0:1789761364.483815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761364.483819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761364.483823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761364.483833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048734720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761364.483842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761364.483848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1789761364.483849:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761364.483854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761364.483859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 814 00000100:00080000:0.0:1789761364.483873:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761364.483879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761364.483885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761364.483900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048734848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761364.483900:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761364/real 1789761364] req@ffff9811c2e2a300 x1876700048734464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761364.483938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761364.483947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761364.483957:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761364.483964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 814 00000100:00080000:0.0:1789761364.483968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761364.483974:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761364.483976:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048734464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761364.483981:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761364.483997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048734976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761364.483998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761364.484012:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761364.484024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048734464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761364.484063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048734592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761364.484084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761364.484095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761364/real 1789761364] req@ffff9811c2e28000 x1876700048734592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761364.484104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048734592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761364.484110:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761364.484113:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761364.484115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048734592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761364.484119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761364.484131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761364.484138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761364/real 1789761364] req@ffff9811c2e2b800 x1876700048734720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761364.484146:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048734720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761364.484151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761364.484153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761364.484155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761364.484158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761364.484169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761364.484175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761364/real 1789761364] req@ffff9811c2e2b480 x1876700048734848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761364.484189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048734848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761364.484194:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761364.484196:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761364.484198:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761364.484202:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761364.484224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761364.484231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761364/real 1789761364] req@ffff9811c2e29500 x1876700048734976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761364.484239:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048734976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761364.484244:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761364.484246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761364.484248:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048734976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761364.739575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761365.379610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761365.763598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761366.403683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761366.787548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761367.427588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761367.811594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761368.451663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761368.835574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761369.475595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761369.603575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761369.603586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761369.603590:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761369.603601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 819 00000100:00080000:3.0:1789761369.603615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761369.603625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761369.603677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048735104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761369.603701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761369.603704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761369.603706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761369.603709:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 819 00000100:00080000:3.0:1789761369.603712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761369.603714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761369.603724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761369.603734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048735232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761369.603743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761369.603754:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048735104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761369.603764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761369.603766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761369.603770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 819 00000100:00080000:3.0:1789761369.603774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761369.603777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761369.603781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761369.603792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048735360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761369.603799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761369.603802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761369.603804:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761369.603806:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 819 00000100:00080000:3.0:1789761369.603809:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761369.603811:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761369.603814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761369.603821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048735488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761369.603827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761369.603830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761369.603831:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761369.603836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761369.603837:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 819 00000100:00080000:3.0:1789761369.603842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761369.603845:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761369.603857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761369.603865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048735616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761369.603904:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761369/real 1789761369] req@ffff9811e1a20380 x1876700048735104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761369.603928:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048735104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761369.603943:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761369.603946:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761369.603949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048735104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761369.603970:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048735232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761369.603989:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761369.603998:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761369/real 1789761369] req@ffff9811e1a21500 x1876700048735232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761369.604006:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048735232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761369.604012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761369.604015:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761369.604017:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048735232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761369.604020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761369.604031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761369.604037:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761369/real 1789761369] req@ffff9811e1a20e00 x1876700048735360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761369.604044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048735360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761369.604049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761369.604051:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761369.604053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761369.604062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761369.604076:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761369.604083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761369/real 1789761369] req@ffff9811e1a20a80 x1876700048735488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761369.604091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048735488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761369.604096:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761369.604098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761369.604100:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761369.604104:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761369.604114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761369.604119:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761369/real 1789761369] req@ffff9811e1a21f80 x1876700048735616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761369.604127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048735616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761369.604132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761369.604133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761369.604135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048735616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761369.859611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761370.499639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761370.883574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761371.523576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761371.907578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761372.547619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761372.931558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761373.571634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761373.955588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761374.595643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761374.723548:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761374.723581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761374.723585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761374.723598:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 824 00000100:00080000:1.0:1789761374.723613:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761374.723623:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761374.723673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048735744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761374.723703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761374.723709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761374.723711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761374.723714:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 824 00000100:00080000:1.0:1789761374.723717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761374.723721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761374.723731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761374.723742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048735872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761374.723749:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048735744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761374.723757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761374.723762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761374.723767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761374.723773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 824 00000100:00080000:1.0:1789761374.723777:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761374.723783:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761374.723788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761374.723803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048736000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761374.723818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761374.723821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761374.723823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761374.723826:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 824 00000100:00080000:1.0:1789761374.723830:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761374.723848:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761374.723852:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761374.723860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761374.723876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048736128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761374.723888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761374.723892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761374.723894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761374.723898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 824 00000100:00080000:1.0:1789761374.723902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761374.723907:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761374/real 1789761374] req@ffff9811c92c4380 x1876700048735744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761374.723908:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761374.723916:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761374.723932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048736256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761374.723939:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048735744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761374.723958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761374.723962:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761374.723966:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048735744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761374.723990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048735872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761374.724009:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761374.724019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761374/real 1789761374] req@ffff9811c92c5500 x1876700048735872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761374.724027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048735872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761374.724033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761374.724036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761374.724038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048735872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761374.724042:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761374.724053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761374.724060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761374/real 1789761374] req@ffff9811c92c5880 x1876700048736000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761374.724078:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048736000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761374.724084:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761374.724086:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761374.724088:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761374.724091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761374.724104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761374.724111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761374/real 1789761374] req@ffff9811c92c4e00 x1876700048736128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761374.724118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048736128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761374.724123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761374.724125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761374.724127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761374.724130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761374.724140:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761374.724145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761374/real 1789761374] req@ffff9811c92c4000 x1876700048736256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761374.724152:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048736256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761374.724166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761374.724169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761374.724171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761374.979564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761375.619580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761376.003582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761376.643618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761377.027568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761377.667579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761378.051608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761378.691648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761379.075601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761379.715689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761379.843592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761379.843604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761379.843610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761379.843624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 829 00000100:00080000:2.0:1789761379.843640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761379.843651:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761379.843697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048736384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761379.843738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761379.843744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761379.843747:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761379.843753:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 829 00000100:00080000:2.0:1789761379.843760:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761379.843768:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761379.843782:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761379.843809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048736384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761379.843818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048736512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761379.843840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761379.843846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761379.843849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761379.843857:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 829 00000100:00080000:2.0:1789761379.843864:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789761379.844030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761379.844077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761379/real 1789761379] req@ffff9811d8639f80 x1876700048736384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761379.844091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048736384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761379.844100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761379.844103:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761379.844106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048736384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761379.844126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048736512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761379.844143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761379.844151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761379/real 1789761379] req@ffff9811d8639500 x1876700048736512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761379.844160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048736512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761379.844165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761379.844168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761379.844170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048736512:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789761379.845176:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761379.845186:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761379.845206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048736640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761379.845229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761379.845233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761379.845236:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761379.845241:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 829 00000100:00080000:2.0:1789761379.845248:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761379.845254:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761379.845262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761379.845269:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761379.845279:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048736768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761379.845296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761379.845305:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761379.845310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761379.845315:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761379.845326:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 829 00000100:00100000:1.0:1789761379.845330:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761379/real 1789761379] req@ffff9811d8639880 x1876700048736640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761379.845334:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761379.845346:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761379.845356:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761379.845381:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048736896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761379.845396:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048736640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761379.845412:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761379.845416:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761379.845420:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761379.845441:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761379.845492:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761379.845503:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761379/real 1789761379] req@ffff9811d863ad80 x1876700048736768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761379.845511:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048736768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761379.845517:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761379.845519:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761379.845521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761379.845526:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761379.845539:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761379.845544:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761379/real 1789761379] req@ffff9811d8639c00 x1876700048736896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761379.845551:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048736896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761379.845566:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761379.845568:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761379.845571:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048736896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761380.099537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761380.739642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761381.123597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761381.763640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761382.147579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761382.787628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761383.171546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761383.811649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761384.195568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761384.835613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761384.963610:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761384.963620:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761384.963625:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761384.963635:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 834 00000100:00080000:0.0:1789761384.963648:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761384.963655:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761384.963696:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048737024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761384.963719:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761384.963722:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761384.963724:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761384.963727:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 834 00000100:00080000:0.0:1789761384.963749:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761384.963753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048737024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761384.963769:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761384.963784:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761384.963803:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048737152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761384.963818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761384.963823:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761384.963825:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761384.963830:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 834 00000100:00080000:0.0:1789761384.963843:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761384.963848:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761384.963853:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761384.963856:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761384.963869:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048737280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761384.963878:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761384.963881:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761384.963882:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761384.963886:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 834 00000100:00080000:0.0:1789761384.963889:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761384.963892:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761384.963896:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761384.963901:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761384/real 1789761384] req@ffff9811c2e29f80 x1876700048737024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761384.963911:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048737408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761384.963920:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048737024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761384.963955:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761384.963955:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761384.963963:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761384.963965:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761384.963966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789761384.963971:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 834 00000100:00080000:0.0:1789761384.963975:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761384.963977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048737024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761384.963980:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761384.963987:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761384.963997:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048737536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761384.964017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048737152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761384.964048:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761384.964064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761384/real 1789761384] req@ffff9811c2e28a80 x1876700048737152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761384.964079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048737152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761384.964097:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761384.964101:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761384.964105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048737152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761384.964110:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761384.964131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761384.964143:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761384/real 1789761384] req@ffff9811c2e28380 x1876700048737280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761384.964156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048737280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761384.964165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761384.964169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761384.964173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761384.964179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761384.964202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761384.964215:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761384/real 1789761384] req@ffff9811c2e28e00 x1876700048737408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761384.964228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048737408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761384.964236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761384.964238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761384.964246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761384.964251:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761384.964271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761384.964283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761384/real 1789761384] req@ffff9811c2e2bb80 x1876700048737536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761384.964296:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048737536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761384.964306:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761384.964309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761384.964313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761385.219626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761385.859615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761386.243578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761386.883619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761387.267594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761387.907646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761388.291558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761388.931589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761389.315583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761389.955589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761390.083597:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761390.083605:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761390.083608:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761390.083617:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 839 00000100:00080000:3.0:1789761390.083626:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761390.083633:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761390.083663:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048737664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761390.083685:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761390.083688:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761390.083690:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761390.083694:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 839 00000100:00080000:3.0:1789761390.083698:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761390.083703:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761390.083714:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761390.083715:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048737664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761390.083751:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048737792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761390.083765:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761390.083769:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:1.0:1789761390.083782:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761390.083788:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761390.083796:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 839 00000100:00080000:3.0:1789761390.083802:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761390.083807:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761390.083814:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761390.083819:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761390/real 1789761390] req@ffff9811e1a23800 x1876700048737664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761390.083830:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048737920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761390.083837:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048737664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761390.083850:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761390.083859:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761390.083864:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761390.083868:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761390.083871:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 839 00000100:00080000:1.0:1789761390.083875:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761390.083877:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761390.083881:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048737664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761390.083886:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761390.083894:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761390.083906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048737792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761390.083906:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048738048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761390.083917:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761390.083920:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761390.083922:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761390.083925:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 839 00000100:00080000:3.0:1789761390.083928:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761390.083931:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761390.083935:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761390.083942:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761390.083955:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048738176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761390.083964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761390/real 1789761390] req@ffff9811e1a20000 x1876700048737792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761390.083980:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048737792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761390.083987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761390.083990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761390.083992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048737792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761390.084003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761390.084019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761390.084027:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761390/real 1789761390] req@ffff9811e1a23480 x1876700048737920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761390.084035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048737920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761390.084040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761390.084042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761390.084044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048737920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761390.084054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761390.084065:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761390.084071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761390/real 1789761390] req@ffff9811e1a23100 x1876700048738048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761390.084089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048738048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761390.084094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761390.084096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761390.084098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761390.084102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761390.084113:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761390.084119:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761390/real 1789761390] req@ffff9811e1a20700 x1876700048738176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761390.084127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048738176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761390.084131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761390.084133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761390.084135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761390.339589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761390.979574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761391.363558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761392.003600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761392.387648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761393.027645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761393.411603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761394.051559:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761394.435590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761395.075619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761395.203549:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761395.203561:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761395.203565:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761395.203577:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 844 00000100:00080000:3.0:1789761395.203590:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761395.203598:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761395.203641:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048738304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761395.203669:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761395.203674:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761395.203675:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761395.203694:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 844 00000100:00080000:3.0:1789761395.203699:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761395.203703:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761395.203714:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761395.203730:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048738432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761395.203732:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048738304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761395.203742:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761395.203746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761395.203748:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761395.203751:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 844 00000100:00080000:3.0:1789761395.203755:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761395.203779:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761395.203784:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761395.203795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048738560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761395.203804:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761395.203808:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761395.203810:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761395.203813:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 844 00000100:00080000:3.0:1789761395.203817:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761395.203821:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761395.203826:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761395.203835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048738688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761395.203845:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761395.203849:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761395.203850:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761395.203854:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 844 00000100:00100000:1.0:1789761395.203858:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048738432:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761395.203858:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761395.203862:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761395.203867:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761395.203878:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048738816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761395.203890:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761395.203931:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761395.203957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761395.459606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761396.099586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761396.483544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761397.123642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761397.507649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761398.147554:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761398.275759:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789761398.275803:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761398.275820:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761398.275869:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761398.275881:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761398.275890:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761398.275901:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789761398.275939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761395/real 1789761398] req@ffff9811e1a22300 x1876700048738816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761398.275966:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048738816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761398.275978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761398.275984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761398.275988:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761398.275997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761395/real 1789761398] req@ffff9811e1a21180 x1876700048738432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761398.276009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048738432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761398.276019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761398.276023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761398.276027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048738432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761398.276035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761395/real 1789761398] req@ffff9811e1a22a00 x1876700048738560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761398.276047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048738560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761398.276078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761398.276082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761398.276085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761398.276092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761395/real 1789761398] req@ffff9811e1a21c00 x1876700048738688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761398.276104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048738688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761398.276111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761398.276114:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761398.276117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048738688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761398.276124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761395/real 1789761398] req@ffff9811e1a21880 x1876700048738304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761398.276135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048738304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761398.276143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761398.276146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761398.276148:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048738304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761398.531544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761399.172811:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761399.555626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761400.195627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761400.323586:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761400.323612:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761400.323616:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761400.323627:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 849 00000100:00080000:3.0:1789761400.323638:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761400.323660:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761400.323694:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048738944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761400.323718:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761400.323721:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761400.323723:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761400.323727:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 849 00000100:00080000:3.0:1789761400.323731:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761400.323736:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761400.323750:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761400.323765:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048739072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761400.323771:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048738944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761400.323789:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761400.323794:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761400.323796:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761400.323802:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 849 00000100:00080000:3.0:1789761400.323807:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761400.323812:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761400.323818:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761400.323831:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048739200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761400.323839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761400.323852:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761400.323857:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761400.323860:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761400.323866:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 849 00000100:00080000:3.0:1789761400.323873:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761400.323877:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761400.323885:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761400.323899:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761400/real 1789761400] req@ffff9811e1a22680 x1876700048738944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761400.323903:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048739328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761400.323926:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761400.323932:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761400.323934:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761400.323939:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 849 00000100:00100000:1.0:1789761400.323943:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048738944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761400.323946:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761400.323956:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789761400.323964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761400.323965:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761400.323974:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761400.323983:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048739456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761400.323984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048738944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761400.324066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048739072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761400.324096:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761400.324111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761400/real 1789761400] req@ffff9811e1a22d80 x1876700048739072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761400.324126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048739072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761400.324135:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761400.324138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761400.324142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048739072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761400.324149:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761400.324171:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761400.324184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761400/real 1789761400] req@ffff9811e1a21880 x1876700048739200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761400.324203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048739200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761400.324225:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761400.324229:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761400.324232:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761400.324239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761400.324263:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761400.324275:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761400/real 1789761400] req@ffff9811e1a21c00 x1876700048739328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761400.324287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048739328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761400.324295:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761400.324299:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761400.324302:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761400.324308:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761400.324328:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761400.324339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761400/real 1789761400] req@ffff9811e1a22a00 x1876700048739456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761400.324351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048739456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761400.324360:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761400.324363:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761400.324374:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761400.579606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761401.219629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761401.603528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761402.243592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761402.627623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761403.267635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761403.651521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761404.291590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761404.675548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761405.315680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761405.443595:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761405.443616:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761405.443621:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761405.443632:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 855 00000100:00080000:1.0:1789761405.443643:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761405.443651:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761405.443689:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048739584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761405.443719:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761405.443728:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761405.443730:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761405.443735:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 855 00000100:00080000:1.0:1789761405.443739:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761405.443744:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761405.443754:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761405.443767:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048739584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761405.443772:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048739712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761405.443786:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761405.443790:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761405.443792:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761405.443813:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 855 00000100:00080000:1.0:1789761405.443828:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761405.443833:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761405.443839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761405.443843:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761405.443860:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048739840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761405.443871:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761405.443875:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761405.443877:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761405.443880:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761405/real 1789761405] req@ffff9811c92c5f80 x1876700048739584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761405.443884:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 855 00000100:00080000:1.0:1789761405.443891:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761405.443899:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761405.443906:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761405.443914:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048739584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761405.443920:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048739968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761405.443936:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761405.443939:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761405.443942:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789761405.443947:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789761405.443951:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048739584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761405.443952:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761405.443962:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 855 00000100:00080000:1.0:1789761405.443966:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761405.443970:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761405.443976:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761405.443977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048739712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761405.443994:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048740096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761405.444012:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761405.444034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761405/real 1789761405] req@ffff9811c92c7480 x1876700048739712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761405.444048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048739712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761405.444054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761405.444057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761405.444059:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048739712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761405.444063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761405.444079:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761405.444100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761405/real 1789761405] req@ffff9811c92c4a80 x1876700048739840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761405.444114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048739840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761405.444123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761405.444127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761405.444131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761405.444136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761405.444153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761405.444161:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761405/real 1789761405] req@ffff9811c92c7800 x1876700048739968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761405.444169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048739968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761405.444174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761405.444177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761405.444180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048739968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761405.444197:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761405.444217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761405.444230:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761405/real 1789761405] req@ffff9811c92c7100 x1876700048740096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761405.444238:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048740096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761405.444261:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761405.444263:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761405.444266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761405.699543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761406.339615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761406.723532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761407.363672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761407.747641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761408.387637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761408.771622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761409.411662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761409.795613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761410.435548:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761410.563610:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761410.563619:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761410.563623:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761410.563631:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 860 00000100:00080000:3.0:1789761410.563659:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761410.563667:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761410.563702:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048740224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761410.563729:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761410.563732:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761410.563733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761410.563736:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 860 00000100:00080000:3.0:1789761410.563739:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761410.563742:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761410.563750:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761410.563762:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048740352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761410.563770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761410.563773:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761410.563775:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761410.563778:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 860 00000100:00080000:3.0:1789761410.563780:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761410.563784:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761410.563787:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761410.563793:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048740480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761410.563799:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761410.563807:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761410.563808:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761410.563812:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 860 00000100:00080000:3.0:1789761410.563815:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761410.563825:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761410.563828:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048740224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761410.563831:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761410.563851:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048740608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761410.563866:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761410.563870:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761410.563872:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761410.563877:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 860 00000100:00080000:3.0:1789761410.563882:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761410.563886:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761410.563893:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761410.563905:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048740736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761410.563917:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761410.563969:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761410/real 1789761410] req@ffff9811e1a21180 x1876700048740224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761410.563986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048740224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761410.563995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761410.563998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761410.564002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048740224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761410.564007:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048740352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761410.564022:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761410.564030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761410/real 1789761410] req@ffff9811e1a22300 x1876700048740352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761410.564038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048740352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761410.564056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761410.564059:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761410.564062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048740352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761410.564066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761410.564090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761410.564098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761410/real 1789761410] req@ffff9811e1a20700 x1876700048740480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761410.564106:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048740480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761410.564112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761410.564114:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761410.564116:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761410.564146:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761410.564160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761410.564167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761410/real 1789761410] req@ffff9811e1a23100 x1876700048740608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761410.564175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048740608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761410.564180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761410.564182:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761410.564189:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761410.564194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761410.564207:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761410.564214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761410/real 1789761410] req@ffff9811e1a23480 x1876700048740736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761410.564222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048740736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761410.564227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761410.564230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761410.564233:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048740736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761410.819601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761411.459584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761411.843584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761412.483544:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761412.867621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761413.507534:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761413.891590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761414.531567:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761414.915588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761415.555567:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761415.683524:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761415.683544:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761415.683548:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761415.683556:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 865 00000100:00080000:3.0:1789761415.683565:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761415.683586:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761415.683613:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048740864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761415.683635:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761415.683638:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761415.683639:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761415.683642:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 865 00000100:00080000:3.0:1789761415.683645:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761415.683648:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761415.683655:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761415.683664:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048740992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761415.683677:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761415.683681:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761415.683684:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761415.683688:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 865 00000100:00080000:3.0:1789761415.683693:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761415.683699:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761415.683705:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761415.683719:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048741120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761415.683728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048740864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761415.683742:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761415.683746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761415.683757:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761415.683764:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 865 00000100:00080000:3.0:1789761415.683770:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761415.683776:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761415.683783:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761415.683800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048741248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761415.683811:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761415.683815:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761415.683821:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761415.683824:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761415.683830:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 865 00000100:00080000:3.0:1789761415.683837:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761415.683843:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761415.683849:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761415.683862:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761415/real 1789761415] req@ffff9811e1a20000 x1876700048740864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761415.683868:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048741376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761415.683898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048740864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761415.683909:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761415.683913:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761415.683929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048740864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761415.683937:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048740992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761415.683962:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761415.683975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761415/real 1789761415] req@ffff9811e1a23800 x1876700048740992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761415.683994:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048740992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761415.684002:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761415.684004:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761415.684006:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048740992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761415.684029:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761415.684048:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761415.684064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761415/real 1789761415] req@ffff9811e1a21f80 x1876700048741120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761415.684079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048741120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761415.684092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761415.684098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761415.684102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761415.684122:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761415.684150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761415.684169:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761415/real 1789761415] req@ffff9811e1a20a80 x1876700048741248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761415.684179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048741248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761415.684186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761415.684189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761415.684193:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761415.684199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761415.684215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761415.684221:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761415/real 1789761415] req@ffff9811e1a20e00 x1876700048741376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761415.684228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048741376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761415.684233:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761415.684235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761415.684237:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761415.939567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761416.579611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761416.963602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761417.603617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761417.987575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761418.627579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761419.011614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761419.651607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761420.035679:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761420.675621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761420.803553:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761420.803561:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761420.803565:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761420.803573:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 870 00000100:00080000:3.0:1789761420.803582:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761420.803588:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761420.803634:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048741504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761420.803665:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761420.803669:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761420.803671:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761420.803676:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 870 00000100:00080000:3.0:1789761420.803680:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761420.803685:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761420.803695:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761420.803713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048741632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761420.803716:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048741504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761420.803736:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761420.803741:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761420.803743:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761420.803748:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 870 00000100:00080000:3.0:1789761420.803753:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761420.803757:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761420.803763:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761420.803773:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048741760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761420.803781:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761420.803783:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761420.803785:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761420.803789:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761420.803794:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 870 00000100:00080000:3.0:1789761420.803798:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761420.803803:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761420.803808:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761420.803822:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048741888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761420.803833:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761420.803838:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761420/real 1789761420] req@ffff9811e1a21500 x1876700048741504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761420.803840:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761420.803845:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761420.803854:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 870 00000100:00100000:0.0:1789761420.803859:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048741504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761420.803860:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761420.803874:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761420.803879:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789761420.803889:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:3.0:1789761420.803889:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761420.803898:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048741504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761420.803909:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048742016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761420.803928:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048741632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761420.803952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761420.803963:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761420/real 1789761420] req@ffff9811e1a20380 x1876700048741632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761420.803972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048741632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761420.803978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761420.803980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761420.803982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048741632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761420.803986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761420.803997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761420.804003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761420/real 1789761420] req@ffff9811e1a20e00 x1876700048741760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761420.804011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048741760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761420.804032:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761420.804036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761420.804040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761420.804046:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761420.804067:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761420.804080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761420/real 1789761420] req@ffff9811e1a20a80 x1876700048741888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761420.804093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048741888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761420.804101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761420.804104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761420.804107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048741888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761420.804114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761420.804133:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761420.804145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761420/real 1789761420] req@ffff9811e1a21f80 x1876700048742016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761420.804157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048742016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761420.804166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761420.804169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761420.804180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761421.059624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761421.699588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761422.083579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761422.723643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761423.107577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761423.747641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761424.131641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761424.771636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761425.155572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761425.795615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761425.923634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761425.923644:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761425.923649:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761425.923660:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 875 00000100:00080000:0.0:1789761425.923672:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761425.923679:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761425.923732:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048742144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761425.923767:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761425.923770:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761425.923772:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761425.923775:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 875 00000100:00080000:0.0:1789761425.923778:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761425.923781:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761425.923791:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761425.923803:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048742272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761425.923813:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761425.923824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048742144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761425.923835:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761425.923837:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761425.923841:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 875 00000100:00080000:0.0:1789761425.923844:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761425.923848:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761425.923852:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761425.923863:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048742400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761425.923871:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761425.923874:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761425.923875:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761425.923877:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 875 00000100:00080000:0.0:1789761425.923887:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761425.923890:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761425.923893:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761425.923900:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761425.923904:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048742528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761425.923911:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761425.923915:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761425.923916:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761425.923920:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 875 00000100:00080000:0.0:1789761425.923926:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761425.923928:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761425.923939:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761425.923952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761425/real 1789761425] req@ffff9811c2e29180 x1876700048742144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761425.923955:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048742656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761425.923983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048742144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761425.923992:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761425.923996:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761425.924000:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048742144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761425.924024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048742272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761425.924041:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761425.924049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761425/real 1789761425] req@ffff9811c2e2aa00 x1876700048742272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761425.924058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048742272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761425.924063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761425.924066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761425.924068:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048742272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761425.924081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761425.924096:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761425.924103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761425/real 1789761425] req@ffff9811c2e2b100 x1876700048742400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761425.924117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048742400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761425.924123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761425.924125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761425.924127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761425.924131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761425.924143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761425.924149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761425/real 1789761425] req@ffff9811c2e29880 x1876700048742528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761425.924157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048742528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761425.924162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761425.924164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761425.924165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761425.924168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761425.924178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761425.924183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761425/real 1789761425] req@ffff9811c2e2ad80 x1876700048742656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761425.924197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048742656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761425.924203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761425.924205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761425.924207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048742656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761426.179549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761426.819615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761427.203573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761427.843595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761428.227608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761428.867673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761429.251582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761429.891656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761430.275604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761430.915634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761431.043566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761431.043579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761431.043583:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761431.043594:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 880 00000100:00080000:1.0:1789761431.043606:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761431.043614:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761431.043653:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048742784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761431.043683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761431.043687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761431.043688:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761431.043692:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 880 00000100:00080000:1.0:1789761431.043696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761431.043701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761431.043711:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761431.043726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048742784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761431.043728:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048742912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761431.043759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761431.043763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761431.043765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761431.043769:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 880 00000100:00080000:1.0:1789761431.043785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761431.043790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761431.043796:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761431.043797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761431.043814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048743040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761431.043825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761431.043829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761431.043831:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761431.043835:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 880 00000100:00080000:1.0:1789761431.043841:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761431.043846:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761431/real 1789761431] req@ffff9811c92c6300 x1876700048742784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761431.043849:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761431.043859:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761431.043873:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048742784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761431.043878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048743168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761431.043893:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761431.043898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761431.043902:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789761431.043905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789761431.043911:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048742784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761431.043911:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761431.043921:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 880 00000100:00080000:1.0:1789761431.043937:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761431.043945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048742912:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761431.043945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761431.043956:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761431.043972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048743296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761431.043983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761431.044004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761431/real 1789761431] req@ffff9811c92c4700 x1876700048742912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761431.044018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048742912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761431.044035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761431.044038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761431.044040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048742912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761431.044045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761431.044061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761431.044069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761431/real 1789761431] req@ffff9811c92c6a00 x1876700048743040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761431.044077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048743040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761431.044083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761431.044085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761431.044087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761431.044090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761431.044103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761431.044112:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761431/real 1789761431] req@ffff9811c92c7100 x1876700048743168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761431.044124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048743168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761431.044132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761431.044135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761431.044146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761431.044165:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761431.044187:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761431.044199:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761431/real 1789761431] req@ffff9811c92c7800 x1876700048743296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761431.044213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048743296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761431.044219:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761431.044221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761431.044223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761431.299541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761431.939623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761432.323602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761432.963611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761433.348650:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761433.987593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761434.371559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761435.011621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761435.395569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761436.035609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761436.163530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761436.163539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761436.163542:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761436.163549:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 885 00000100:00080000:1.0:1789761436.163558:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761436.163564:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761436.163594:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048743424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761436.163646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761436.163649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761436.163650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761436.163653:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 885 00000100:00080000:1.0:1789761436.163656:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761436.163658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761436.163667:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761436.163677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048743552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761436.163686:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048743424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761436.163704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761436.163707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761436.163708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761436.163712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 885 00000100:00080000:1.0:1789761436.163715:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761436.163718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761436.163722:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761436.163732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048743680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761436.163738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761436.163741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761436.163742:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761436.163745:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 885 00000100:00080000:1.0:1789761436.163747:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761436.163749:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761436.163753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761436.163759:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761436.163767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048743808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761436.163778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761436.163782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761436.163784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761436.163788:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 885 00000100:00080000:1.0:1789761436.163801:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761436.163805:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761436/real 1789761436] req@ffff9811c92c4a80 x1876700048743424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761436.163810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761436.163822:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761436.163835:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048743424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761436.163840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048743936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761436.163862:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761436.163870:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761436.163883:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048743424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761436.163910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048743552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761436.163936:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761436.163951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761436/real 1789761436] req@ffff9811c92c7480 x1876700048743552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761436.163966:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048743552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761436.163976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761436.163980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761436.163983:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048743552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761436.163990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761436.164007:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761436.164016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761436/real 1789761436] req@ffff9811c92c5f80 x1876700048743680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761436.164023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048743680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761436.164029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761436.164031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761436.164033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761436.164037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761436.164047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761436.164060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761436/real 1789761436] req@ffff9811c92c4000 x1876700048743808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761436.164068:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048743808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761436.164073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761436.164075:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761436.164077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761436.164081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761436.164091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761436.164097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761436/real 1789761436] req@ffff9811c92c4e00 x1876700048743936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761436.164104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048743936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761436.164109:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761436.164110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761436.164112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048743936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761436.419587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761437.059593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761437.443523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761438.083620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761438.467572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761439.107634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761439.491578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761440.131658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761440.515657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761441.155660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761441.283580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761441.283591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761441.283595:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761441.283606:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 890 00000100:00080000:2.0:1789761441.283618:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761441.283625:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761441.283695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048744064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761441.283727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761441.283730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761441.283731:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761441.283734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 890 00000100:00080000:2.0:1789761441.283738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761441.283741:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761441.283751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761441.283762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048744192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761441.283772:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761441.283775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761441.283776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761441.283782:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 890 00000100:00100000:0.0:1789761441.283784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048744064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761441.283784:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761441.283790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761441.283796:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761441.283803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048744320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761441.283810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761441.283813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761441.283814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761441.283823:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 890 00000100:00080000:2.0:1789761441.283826:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761441.283829:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761441.283834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761441.283844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048744448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761441.283851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761441.283854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761441.283855:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761441.283858:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 890 00000100:00080000:2.0:1789761441.283860:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761441.283862:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761441.283864:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761441.283868:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761441.283877:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048744576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761441.283919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761441/real 1789761441] req@ffff9811d863b100 x1876700048744064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761441.283937:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048744064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761441.283946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761441.283950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761441.283953:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048744064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761441.283975:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048744192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761441.283992:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761441.284001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761441/real 1789761441] req@ffff9811d863bb80 x1876700048744192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761441.284015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048744192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761441.284025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761441.284028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761441.284032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048744192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761441.284039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761441.284060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761441.284070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761441/real 1789761441] req@ffff9811d8639180 x1876700048744320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761441.284079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048744320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761441.284095:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761441.284098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761441.284100:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761441.284105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761441.284120:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761441.284128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761441/real 1789761441] req@ffff9811d8639c00 x1876700048744448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761441.284136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048744448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761441.284141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761441.284143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761441.284145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761441.284149:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761441.284159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761441.284164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761441/real 1789761441] req@ffff9811d863ad80 x1876700048744576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761441.284172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048744576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761441.284177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761441.284179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761441.284186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761441.539595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761442.179648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761442.563606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761443.203576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761443.587598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761444.227576:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761444.611565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761445.251612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761445.635687:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761446.275630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761446.403583:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761446.403594:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761446.403598:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761446.403608:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 895 00000100:00080000:0.0:1789761446.403618:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761446.403625:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761446.403662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048744704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761446.403693:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761446.403698:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761446.403700:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761446.403705:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 895 00000100:00080000:0.0:1789761446.403710:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761446.403715:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761446.403726:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761446.403742:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048744704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761446.403744:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048744832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761446.403770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761446.403775:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761446.403792:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761446.403798:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 895 00000100:00080000:0.0:1789761446.403817:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761446.403822:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761446.403829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761446.403833:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761446.403854:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048744960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761446.403868:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761446.403872:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761446.403874:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761446.403880:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 895 00000100:00100000:1.0:1789761446.403886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761446/real 1789761446] req@ffff9811c2e29c00 x1876700048744704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761446.403892:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761446.403905:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761446.403915:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761446.403922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048744704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761446.403934:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048745088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761446.403943:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761446.403951:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789761446.403952:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761446.403957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048744704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761446.403960:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761446.403965:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761446.403972:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 895 00000100:00080000:0.0:1789761446.403976:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761446.403981:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761446.403990:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761446.403991:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048744832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761446.404020:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048745216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761446.404025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761446.404040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761446/real 1789761446] req@ffff9811c2e2a680 x1876700048744832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761446.404069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048744832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761446.404077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761446.404080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761446.404082:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048744832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761446.404088:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761446.404108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761446.404121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761446/real 1789761446] req@ffff9811c2e28700 x1876700048744960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761446.404134:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048744960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761446.404143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761446.404147:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761446.404151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048744960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761446.404158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761446.404183:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761446.404195:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761446/real 1789761446] req@ffff9811c2e2ad80 x1876700048745088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761446.404209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048745088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761446.404232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761446.404236:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761446.404239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761446.404255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761446.404276:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761446.404288:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761446/real 1789761446] req@ffff9811c2e29880 x1876700048745216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761446.404316:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048745216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761446.404325:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761446.404327:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761446.404330:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761446.659544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761447.299588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761447.683560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761448.323586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761448.707554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761449.347563:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761449.731583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761450.371694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761450.755560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761451.395638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761451.523620:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761451.523633:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761451.523639:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761451.523650:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 901 00000100:00080000:0.0:1789761451.523663:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761451.523671:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761451.523713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048745344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761451.523739:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761451.523743:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761451.523744:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761451.523748:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 901 00000100:00080000:0.0:1789761451.523752:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761451.523755:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761451.523766:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761451.523775:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048745344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761451.523779:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048745472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761451.523792:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761451.523812:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761451.523814:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761451.523818:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 901 00000100:00080000:0.0:1789761451.523823:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761451.523827:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761451.523832:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761451.523844:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048745600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761451.523853:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761451.523857:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761451.523859:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761451.523860:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761451.523864:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 901 00000100:00080000:0.0:1789761451.523868:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761451.523871:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761451.523875:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761451.523883:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048745728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761451.523890:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761451.523893:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761451.523894:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761451.523897:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 901 00000100:00080000:0.0:1789761451.523899:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761451.523902:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761451.523913:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761451/real 1789761451] req@ffff9811c2e2b100 x1876700048745344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761451.523931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048745344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761451.523933:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761451.523946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761451.523950:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048745856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761451.523953:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761451.523960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048745344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761451.524006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048745472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761451.524025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761451.524035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761451/real 1789761451] req@ffff9811c2e2aa00 x1876700048745472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761451.524044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048745472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761451.524050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761451.524053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761451.524055:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048745472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761451.524058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761451.524070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761451.524076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761451/real 1789761451] req@ffff9811c2e29180 x1876700048745600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761451.524084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048745600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761451.524099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761451.524102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761451.524104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761451.524108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761451.524123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761451.524130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761451/real 1789761451] req@ffff9811c2e2bb80 x1876700048745728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761451.524137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048745728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761451.524142:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761451.524144:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761451.524146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761451.524149:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761451.524160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761451.524165:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761451/real 1789761451] req@ffff9811c2e28e00 x1876700048745856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761451.524173:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048745856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761451.524177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761451.524187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761451.524195:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048745856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761451.779607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761452.419581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761452.803588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761453.443567:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761453.827614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761454.467578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761454.851558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761455.491627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761455.875584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761456.515620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761456.643582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761456.643593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761456.643598:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761456.643610:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 906 00000100:00080000:0.0:1789761456.643621:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761456.643629:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761456.643672:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048745984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761456.643699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761456.643702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761456.643703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761456.643706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 906 00000100:00080000:0.0:1789761456.643709:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761456.643711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761456.643720:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761456.643730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048746112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761456.643767:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048745984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761456.643772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761456.643778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761456.643779:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761456.643785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 906 00000100:00080000:0.0:1789761456.643789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761456.643792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761456.643797:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761456.643816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048746240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761456.643830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761456.643835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761456.643838:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761456.643845:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761456.643846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 906 00000100:00080000:0.0:1789761456.643855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761456.643861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761456.643870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761456.643884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048746368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761456.643894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761456.643895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761456/real 1789761456] req@ffff9811c2e28380 x1876700048745984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761456.643904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761456.643911:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761456.643926:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 906 00000100:00080000:0.0:1789761456.643937:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761456.643947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761456.643948:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048745984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761456.643953:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761456.643971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048746496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761456.643976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761456.643986:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761456.643992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048745984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761456.644022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048746112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761456.644049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761456.644062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761456/real 1789761456] req@ffff9811c2e28a80 x1876700048746112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761456.644077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048746112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761456.644087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761456.644091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761456.644107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048746112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761456.644114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761456.644139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761456.644153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761456/real 1789761456] req@ffff9811c2e29f80 x1876700048746240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761456.644166:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048746240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761456.644175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761456.644179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761456.644182:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761456.644190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761456.644211:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761456.644220:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761456/real 1789761456] req@ffff9811c2e29500 x1876700048746368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761456.644245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048746368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761456.644250:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761456.644252:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761456.644254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761456.644259:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761456.644274:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761456.644287:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761456/real 1789761456] req@ffff9811c2e2b480 x1876700048746496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761456.644296:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048746496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761456.644301:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761456.644303:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761456.644305:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761456.899569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761457.539634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761457.923623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761458.563625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761458.947564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761459.587604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761459.971570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761460.611599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761460.995539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761461.635591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761461.763572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761461.763580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761461.763582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761461.763590:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 911 00000100:00080000:0.0:1789761461.763600:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761461.763606:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761461.763638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048746624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761461.763666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761461.763670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761461.763672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761461.763677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 911 00000100:00080000:0.0:1789761461.763680:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761461.763685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761461.763694:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761461.763719:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048746624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761461.763722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048746752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761461.763733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761461.763737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761461.763738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761461.763742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 911 00000100:00080000:0.0:1789761461.763745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761461.763749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761461.763754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761461.763762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048746880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761461.763770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761461.763773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761461.763775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761461.763778:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 911 00000100:00080000:0.0:1789761461.763781:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761461.763785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761461.763789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761461.763799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048747008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761461.763807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761461.763810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761461.763812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761461.763822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 911 00000100:00100000:2.0:1789761461.763822:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048746752:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789761461.763826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761461.763831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761461.763836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761461.763848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048747136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761461.763855:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761461.763881:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761461.763910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761462.019576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761462.659591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761463.043734:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761463.683616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761464.067660:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761464.707575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761464.835793:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789761464.835854:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761464.835876:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789761464.835932:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789761464.835946:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789761464.835963:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789761464.835975:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761464.836022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761461/real 1789761464] req@ffff9811c2e29500 x1876700048747136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761464.836062:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048747136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761464.836075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761464.836080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761464.836105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761464.836115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761461/real 1789761464] req@ffff9811c2e28000 x1876700048746752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761464.836132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048746752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761464.836141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761464.836146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761464.836150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048746752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761464.836158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761461/real 1789761464] req@ffff9811c2e2a300 x1876700048746880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761464.836187:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048746880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761464.836195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761464.836199:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761464.836203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048746880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761464.836210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761461/real 1789761464] req@ffff9811c2e2b480 x1876700048747008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761464.836222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048747008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761464.836229:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761464.836232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761464.836242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761464.836249:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761461/real 1789761464] req@ffff9811c2e2b800 x1876700048746624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761464.836263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048746624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761464.836270:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761464.836273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761464.836276:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048746624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761465.091609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761465.731679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761466.115596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761466.755604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761466.883688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761466.883698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761466.883718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761466.883729:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 916 00000100:00080000:1.0:1789761466.883741:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761466.883748:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761466.883786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048747264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761466.883818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761466.883823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761466.883826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761466.883830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 916 00000100:00080000:1.0:1789761466.883834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761466.883838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761466.883850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761466.883865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048747392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761466.883879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761466.883884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761466.883886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761466.883893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 916 00000100:00100000:2.0:1789761466.883893:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048747264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761466.883898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761466.883906:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761466.883911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761466.883925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048747520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761466.883964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761466.883969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761466.883971:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761466.883977:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 916 00000100:00080000:1.0:1789761466.883983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761466.883989:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789761466.883995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761466.883999:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761466.884016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048747648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761466.884025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761466.884029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761466.884031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761466.884034:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 916 00000100:00080000:1.0:1789761466.884039:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761466.884044:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761466.884049:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761466.884056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761466/real 1789761466] req@ffff9811c92c5880 x1876700048747264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761466.884063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048747776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761466.884098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048747264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761466.884113:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761466.884120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761466.884126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048747264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761466.884179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048747392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761466.884210:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761466.884228:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761466/real 1789761466] req@ffff9811c92c5500 x1876700048747392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761466.884259:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048747392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761466.884269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761466.884273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761466.884277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048747392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761466.884284:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761466.884310:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761466.884324:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761466/real 1789761466] req@ffff9811c92c4380 x1876700048747520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761466.884337:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048747520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761466.884345:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761466.884348:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761466.884351:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761466.884357:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761466.884378:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761466.884388:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761466/real 1789761466] req@ffff9811c92c6680 x1876700048747648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761466.884400:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048747648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761466.884419:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761466.884423:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761466.884426:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761466.884432:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761466.884509:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761466.884522:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761466/real 1789761466] req@ffff9811c92c5c00 x1876700048747776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761466.884536:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048747776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761466.884545:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761466.884549:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761466.884553:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048747776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761467.139582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761467.779633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761468.163578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761468.803607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761469.187551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761469.827642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761470.211534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761470.851612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761471.235591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761471.875631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761472.003545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761472.003560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761472.003589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761472.003602:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 921 00000100:00080000:1.0:1789761472.003622:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761472.003631:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761472.003676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048747904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761472.003703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761472.003706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761472.003708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761472.003711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 921 00000100:00080000:1.0:1789761472.003714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761472.003717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761472.003727:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761472.003736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048748032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761472.003745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761472.003749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761472.003750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761472.003752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 921 00000100:00080000:1.0:1789761472.003755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761472.003760:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048747904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761472.003761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761472.003765:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761472.003776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048748160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761472.003794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761472.003797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761472.003798:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761472.003802:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 921 00000100:00080000:1.0:1789761472.003805:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761472.003808:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761472.003812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761472.003823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048748288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761472.003830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761472.003833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761472.003834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761472.003836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 921 00000100:00080000:1.0:1789761472.003838:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761472.003840:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761472.003843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761472.003850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761472.003852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048748416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761472.003906:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761472/real 1789761472] req@ffff9811c92c6d80 x1876700048747904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761472.003929:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048747904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761472.003941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761472.003954:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761472.003959:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048747904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761472.003968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048748032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761472.003994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761472.004008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761472/real 1789761472] req@ffff9811c92c5180 x1876700048748032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761472.004023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048748032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761472.004033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761472.004037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761472.004041:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048748032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761472.004069:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761472.004090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761472.004101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761472/real 1789761472] req@ffff9811c92c7b80 x1876700048748160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761472.004129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048748160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761472.004151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761472.004155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761472.004159:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761472.004166:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761472.004198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761472.004211:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761472/real 1789761472] req@ffff9811c92c5c00 x1876700048748288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761472.004224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048748288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761472.004234:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761472.004238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761472.004242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761472.004249:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761472.004268:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761472.004278:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761472/real 1789761472] req@ffff9811c92c6680 x1876700048748416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761472.004292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048748416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761472.004301:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761472.004305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761472.004309:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761472.259560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761472.899636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761473.283536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761473.923632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761474.307537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761474.947587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761475.331569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761475.971629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761476.355539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761476.995560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761477.123580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761477.123594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761477.123600:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761477.123613:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 926 00000100:00080000:0.0:1789761477.123627:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761477.123637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761477.123685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048748544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761477.123736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761477.123740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761477.123743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761477.123748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 926 00000100:00080000:0.0:1789761477.123753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761477.123758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761477.123770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761477.123784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048748672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761477.123791:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048748544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761477.123804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761477.123810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761477.123812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761477.123817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 926 00000100:00080000:0.0:1789761477.123821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761477.123826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761477.123832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761477.123844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048748800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761477.123856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761477.123860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761477.123863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761477.123868:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 926 00000800:00000100:2.0:1789761477.123878:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761477.123881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761477.123889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761477.123897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761477.123910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048748928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761477.123922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761477.123926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761477.123928:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761477.123933:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 926 00000100:00080000:0.0:1789761477.123938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761477.123939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761477/real 1789761477] req@ffff9811c2e29f80 x1876700048748544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761477.123948:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761477.123956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761477.123964:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048748544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761477.123971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048749056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761477.123977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761477.123986:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761477.123994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048748544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761477.124025:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048748672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761477.124051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761477.124064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761477/real 1789761477] req@ffff9811c2e28a80 x1876700048748672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761477.124090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048748672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761477.124097:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761477.124100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761477.124102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048748672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761477.124107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761477.124123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761477.124131:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761477/real 1789761477] req@ffff9811c2e28380 x1876700048748800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761477.124140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048748800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761477.124156:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761477.124159:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761477.124161:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761477.124166:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761477.124182:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761477.124189:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761477/real 1789761477] req@ffff9811c2e28e00 x1876700048748928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761477.124197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048748928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761477.124202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761477.124205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761477.124207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048748928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761477.124210:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761477.124220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761477.124226:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761477/real 1789761477] req@ffff9811c2e2bb80 x1876700048749056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761477.124232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048749056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761477.124236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761477.124238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761477.124245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761477.379581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761478.019720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761478.403581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761479.043629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761479.427569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761480.067592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761480.451583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761481.091551:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761481.475557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761482.115609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761482.243577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761482.243586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761482.243589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761482.243598:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 931 00000100:00080000:0.0:1789761482.243608:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761482.243613:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761482.243660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048749184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761482.243678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761482.243681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761482.243683:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761482.243687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 931 00000100:00080000:0.0:1789761482.243690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761482.243692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761482.243699:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048749184:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761482.243700:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761482.243714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048749312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761482.243723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761482.243725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761482.243727:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761482.243729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 931 00000100:00080000:0.0:1789761482.243732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761482.243734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761482.243737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761482.243745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048749440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761482.243751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761482.243754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761482.243755:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761482.243765:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 931 00000800:00000100:2.0:1789761482.243766:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761482.243769:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761482.243773:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761482.243778:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761482.243786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048749568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761482.243792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761482.243794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761482.243795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761482.243797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 931 00000100:00080000:0.0:1789761482.243800:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761482.243802:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761482.243805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761482.243821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048749696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761482.243821:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761482/real 1789761482] req@ffff9811c2e29180 x1876700048749184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761482.243855:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048749184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761482.243867:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761482.243871:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761482.243874:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048749184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761482.243894:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048749312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761482.243912:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761482.243921:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761482/real 1789761482] req@ffff9811c2e2aa00 x1876700048749312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761482.243936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048749312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761482.243941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761482.243944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761482.243946:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048749312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761482.243949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761482.243970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761482.243977:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761482/real 1789761482] req@ffff9811c2e2b100 x1876700048749440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761482.243985:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048749440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761482.243991:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761482.243993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761482.243995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761482.243998:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761482.244008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761482.244014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761482/real 1789761482] req@ffff9811c2e29880 x1876700048749568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761482.244021:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048749568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761482.244025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761482.244027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761482.244029:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761482.244050:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761482.244063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761482.244070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761482/real 1789761482] req@ffff9811c2e2ad80 x1876700048749696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761482.244088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048749696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761482.244094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761482.244096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761482.244098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048749696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761482.499571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761483.139612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761483.523580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761484.163583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761484.547593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761485.187617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761485.571590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761486.211636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761486.595609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761487.235607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761487.363550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761487.363565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761487.363571:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761487.363585:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 936 00000100:00080000:3.0:1789761487.363601:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761487.363612:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761487.363652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048749824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761487.363686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761487.363692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761487.363694:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761487.363700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 936 00000100:00080000:3.0:1789761487.363706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761487.363711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761487.363721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761487.363727:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048749824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761487.363736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048749952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761487.363749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761487.363753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761487.363754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761487.363758:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 936 00000100:00080000:3.0:1789761487.363762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761487.363765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761487.363777:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761487.363787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048750080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761487.363802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761487.363807:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761487.363811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761487.363815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761487.363819:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 936 00000100:00080000:3.0:1789761487.363824:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761487.363862:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761487/real 1789761487] req@ffff9811e1a23800 x1876700048749824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761487.363891:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048749824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761487.363892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761487.363905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761487.363906:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761487.363912:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761487.363915:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048749824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761487.363937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048750208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761487.363946:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048749952:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761487.363959:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761487.363964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761487.363967:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761487.363973:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 936 00000800:00000100:0.0:1789761487.363975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761487.363982:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761487.363990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761487.363991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761487/real 1789761487] req@ffff9811e1a20000 x1876700048749952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761487.364008:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761487.364016:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048749952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761487.364024:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761487.364026:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048750336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761487.364029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761487.364035:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048749952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761487.364042:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761487.364064:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761487.364076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761487/real 1789761487] req@ffff9811e1a23480 x1876700048750080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761487.364090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048750080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761487.364112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761487.364117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761487.364121:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761487.364139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761487.364159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761487.364174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761487/real 1789761487] req@ffff9811e1a23100 x1876700048750208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761487.364185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048750208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761487.364192:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761487.364195:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761487.364197:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761487.364205:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761487.364220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761487.364231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761487/real 1789761487] req@ffff9811e1a20700 x1876700048750336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761487.364241:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048750336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761487.364247:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761487.364251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761487.364259:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761487.619594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761488.259583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761488.643565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761489.283657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761489.667600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761490.307598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761490.691588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761491.332559:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761491.715528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761492.355567:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761492.483538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761492.483546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761492.483549:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761492.483557:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 942 00000100:00080000:3.0:1789761492.483566:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761492.483572:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761492.483610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048750464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761492.483639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761492.483644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761492.483646:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761492.483651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 942 00000100:00080000:3.0:1789761492.483655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761492.483660:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761492.483670:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761492.483683:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048750464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761492.483696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048750592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761492.483711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761492.483714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761492.483716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761492.483720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 942 00000100:00080000:3.0:1789761492.483723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761492.483726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761492.483740:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789761492.483755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761492.483772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048750720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761492.483777:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761492/real 1789761492] req@ffff9811e1a22300 x1876700048750464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761492.483800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761492.483805:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048750464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761492.483807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761492.483812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761492.483821:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761492.483821:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 942 00000100:00080000:0.0:1789761492.483828:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761492.483830:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761492.483834:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048750464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761492.483839:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761492.483847:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761492.483857:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048750592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761492.483862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048750848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761492.483876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761492.483879:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761492.483883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761492.483887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761492.483894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 942 00000100:00100000:0.0:1789761492.483895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761492/real 1789761492] req@ffff9811e1a21180 x1876700048750592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761492.483899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761492.483907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761492.483913:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761492.483921:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048750592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761492.483927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048750976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761492.483934:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761492.483942:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761492.483968:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048750592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761492.483975:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761492.483995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761492.484008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761492/real 1789761492] req@ffff9811e1a22a00 x1876700048750720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761492.484022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048750720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761492.484030:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761492.484033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761492.484037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761492.484052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761492.484074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761492.484084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761492/real 1789761492] req@ffff9811e1a21c00 x1876700048750848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761492.484092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048750848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761492.484114:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761492.484116:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761492.484118:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761492.484122:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761492.484135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761492.484149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761492/real 1789761492] req@ffff9811e1a21880 x1876700048750976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761492.484157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048750976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761492.484163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761492.484165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761492.484167:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048750976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761492.739549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761493.379571:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761493.763583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761494.403625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761494.787569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761495.427653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761495.811538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761496.451678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761496.835580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761497.475653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761497.603602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761497.603615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761497.603620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761497.603633:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 947 00000100:00080000:0.0:1789761497.603647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761497.603657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761497.603703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048751104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761497.603731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761497.603734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761497.603736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761497.603739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 947 00000100:00080000:0.0:1789761497.603742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761497.603762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761497.603772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761497.603785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048751232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761497.603798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761497.603803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761497.603805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761497.603809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 947 00000100:00080000:0.0:1789761497.603814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761497.603820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761497.603831:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048751104:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761497.603836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761497.603857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048751360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761497.603871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761497.603875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761497.603877:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761497.603883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 947 00000100:00080000:0.0:1789761497.603888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761497.603893:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761497.603901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761497.603911:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761497.603934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048751488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761497.603945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761497.603953:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761497.603954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761497.603958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 947 00000100:00080000:0.0:1789761497.603963:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761497.603967:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761497.603973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761497.603979:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761497/real 1789761497] req@ffff9811c2e28700 x1876700048751104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761497.603985:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048751616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761497.604028:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048751104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761497.604041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761497.604046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761497.604050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048751104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761497.604057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048751232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761497.604078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761497.604086:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761497/real 1789761497] req@ffff9811c2e2a680 x1876700048751232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761497.604095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048751232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761497.604117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761497.604120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761497.604122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048751232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761497.604143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761497.604159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761497.604167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761497/real 1789761497] req@ffff9811c2e29c00 x1876700048751360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761497.604174:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048751360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761497.604180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761497.604182:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761497.604184:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761497.604188:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761497.604198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761497.604204:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761497/real 1789761497] req@ffff9811c2e2ad80 x1876700048751488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761497.604211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048751488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761497.604216:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761497.604217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761497.604224:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761497.604229:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761497.604243:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761497.604250:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761497/real 1789761497] req@ffff9811c2e29880 x1876700048751616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761497.604257:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048751616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761497.604262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761497.604264:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761497.604266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048751616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761497.859531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761498.499618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761498.883551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761499.523591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761499.907560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761500.547639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761500.931566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761501.571630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761501.955566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761502.595653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761502.723552:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761502.723563:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761502.723568:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761502.723580:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 952 00000100:00080000:0.0:1789761502.723592:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761502.723600:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761502.723645:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048751744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761502.723677:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761502.723680:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761502.723681:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761502.723685:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 952 00000100:00080000:0.0:1789761502.723705:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761502.723708:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761502.723718:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761502.723730:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048751872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761502.723740:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761502.723743:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761502.723744:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761502.723747:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 952 00000100:00080000:0.0:1789761502.723750:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761502.723752:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761502.723755:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761502.723763:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048752000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761502.723773:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761502.723776:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761502.723779:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761502.723779:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048751744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761502.723790:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 952 00000100:00080000:0.0:1789761502.723800:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761502.723804:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761502.723808:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761502.723817:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048752128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761502.723833:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761502.723836:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761502.723838:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761502.723841:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 952 00000100:00080000:0.0:1789761502.723843:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761502.723848:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761502.723853:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761502.723865:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048752256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761502.723888:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761502.723939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761502/real 1789761502] req@ffff9811c2e2b100 x1876700048751744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761502.723955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048751744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761502.723964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761502.723968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761502.723971:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048751744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761502.723977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048751872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761502.723994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761502.724015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761502/real 1789761502] req@ffff9811c2e2aa00 x1876700048751872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761502.724023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048751872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761502.724029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761502.724031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761502.724033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048751872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761502.724053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761502.724066:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761502.724072:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761502/real 1789761502] req@ffff9811c2e29180 x1876700048752000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761502.724080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048752000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761502.724085:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761502.724087:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761502.724089:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761502.724093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761502.724103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761502.724108:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761502/real 1789761502] req@ffff9811c2e2bb80 x1876700048752128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761502.724116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048752128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761502.724125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761502.724127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761502.724129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761502.724133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761502.724147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761502.724153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761502/real 1789761502] req@ffff9811c2e28e00 x1876700048752256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761502.724160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048752256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761502.724165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761502.724167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761502.724169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761502.979553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761503.619648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761504.003608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761504.643626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761505.027544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761505.667592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761506.051616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761506.691636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761507.075583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761507.715658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761507.843577:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761507.843593:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761507.843598:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761507.843612:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 957 00000100:00080000:0.0:1789761507.843628:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761507.843638:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761507.843686:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048752384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761507.843728:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761507.843734:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761507.843756:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761507.843762:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 957 00000100:00080000:0.0:1789761507.843767:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761507.843773:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761507.843784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048752384:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761507.843786:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761507.843804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048752512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761507.843820:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761507.843826:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761507.843829:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761507.843835:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 957 00000100:00080000:0.0:1789761507.843853:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761507.843860:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761507.843867:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761507.843883:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048752640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761507.843896:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761507.843899:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761507.843901:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761507.843905:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 957 00000100:00080000:0.0:1789761507.843911:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761507.843916:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761507.843922:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761507.843942:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048752768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761507.843952:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761507.843953:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761507.843958:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761507.843960:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761507.843965:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 957 00000100:00080000:0.0:1789761507.843969:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761507.843972:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761507.843975:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761507.843982:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048752896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761507.844167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761507/real 1789761507] req@ffff9811c2e28380 x1876700048752384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761507.844185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048752384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761507.844196:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761507.844199:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761507.844203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048752384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761507.844225:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048752512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761507.844245:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761507.844255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761507/real 1789761507] req@ffff9811c2e28a80 x1876700048752512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761507.844263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048752512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761507.844269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761507.844272:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761507.844274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048752512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761507.844278:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761507.844290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761507.844296:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761507/real 1789761507] req@ffff9811c2e29f80 x1876700048752640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761507.844314:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048752640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761507.844321:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761507.844323:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761507.844325:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761507.844328:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761507.844342:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761507.844349:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761507/real 1789761507] req@ffff9811c2e2b800 x1876700048752768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761507.844365:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048752768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761507.844371:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761507.844373:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761507.844375:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761507.844381:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761507.844401:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761507.844413:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761507/real 1789761507] req@ffff9811c2e2b480 x1876700048752896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761507.844426:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048752896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761507.844448:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761507.844452:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761507.844457:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048752896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761508.099607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761508.739677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761509.123557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761509.763663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761510.147600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761510.787671:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761511.171516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761511.811614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761512.195618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761512.835595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761512.963535:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761512.963544:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761512.963547:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761512.963555:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 962 00000100:00080000:0.0:1789761512.963567:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761512.963576:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761512.963626:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048753024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761512.963648:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761512.963663:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761512.963664:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761512.963668:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 962 00000100:00080000:0.0:1789761512.963671:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761512.963675:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761512.963684:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761512.963696:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048753152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761512.963707:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761512.963710:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761512.963711:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761512.963716:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 962 00000100:00080000:0.0:1789761512.963719:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761512.963727:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761512.963729:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048753024:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761512.963731:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761512.963743:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048753280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761512.963751:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761512.963760:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761512.963761:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761512.963765:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 962 00000100:00080000:0.0:1789761512.963768:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761512.963770:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761512.963779:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761512.963789:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048753408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761512.963795:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761512.963798:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761512.963799:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761512.963814:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 962 00000100:00080000:0.0:1789761512.963820:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761512.963823:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761512.963828:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761512.963839:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048753536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761512.963856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761512.963916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761512/real 1789761512] req@ffff9811c2e2a300 x1876700048753024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761512.963939:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048753024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761512.963952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761512.963958:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761512.963962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048753024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761512.963996:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048753152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761512.964019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761512.964029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761512/real 1789761512] req@ffff9811c2e28000 x1876700048753152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761512.964038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048753152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761512.964045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761512.964047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761512.964050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048753152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761512.964068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761512.964083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761512.964090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761512/real 1789761512] req@ffff9811c2e29500 x1876700048753280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761512.964098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048753280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761512.964104:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761512.964106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761512.964108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761512.964112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761512.964123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761512.964129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761512/real 1789761512] req@ffff9811c2e2b480 x1876700048753408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761512.964135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048753408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761512.964140:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761512.964143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761512.964144:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761512.964148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761512.964158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761512.964164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761512/real 1789761512] req@ffff9811c2e2b800 x1876700048753536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761512.964176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048753536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761512.964182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761512.964184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761512.964186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761513.219542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761513.859640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761514.243570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761514.883606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761515.267563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761515.907622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761516.291622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761516.931650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761517.315513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761517.955630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761518.083555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761518.083567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761518.083571:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761518.083582:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 967 00000100:00080000:0.0:1789761518.083594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761518.083602:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761518.083645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048753664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761518.083694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761518.083697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761518.083699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761518.083702:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 967 00000100:00080000:0.0:1789761518.083705:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761518.083708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761518.083718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761518.083729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048753792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761518.083737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761518.083740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761518.083741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761518.083743:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 967 00000100:00080000:0.0:1789761518.083745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761518.083748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761518.083754:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048753664:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761518.083758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761518.083780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048753920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761518.083788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761518.083791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761518.083793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761518.083795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 967 00000100:00080000:0.0:1789761518.083804:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761518.083808:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761518.083812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761518.083822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048754048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761518.083828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761518.083831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761518.083832:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761518.083834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 967 00000100:00080000:0.0:1789761518.083840:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761518.083842:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761518.083845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761518.083854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048754176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761518.083857:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761518.083918:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761518/real 1789761518] req@ffff9811c2e29f80 x1876700048753664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761518.083938:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048753664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761518.083951:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761518.083956:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761518.083960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048753664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761518.083969:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048753792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761518.083992:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761518.084006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761518/real 1789761518] req@ffff9811c2e28a80 x1876700048753792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761518.084037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048753792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761518.084047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761518.084051:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761518.084065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048753792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761518.084095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761518.084117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761518.084130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761518/real 1789761518] req@ffff9811c2e28380 x1876700048753920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761518.084144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048753920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761518.084153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761518.084157:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761518.084161:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048753920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761518.084167:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761518.084187:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761518.084197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761518/real 1789761518] req@ffff9811c2e28e00 x1876700048754048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761518.084210:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048754048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761518.084233:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761518.084237:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761518.084241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761518.084247:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761518.084268:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761518.084280:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761518/real 1789761518] req@ffff9811c2e2bb80 x1876700048754176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761518.084293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048754176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761518.084303:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761518.084307:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761518.084310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761518.339527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761518.979620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761519.363609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761520.003622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761520.387633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761521.027635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761521.411634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761522.051662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761522.435550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761523.075623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761523.203583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761523.203592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761523.203595:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761523.203603:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 972 00000100:00080000:0.0:1789761523.203611:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761523.203617:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761523.203663:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048754304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761523.203686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761523.203689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761523.203690:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761523.203693:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 972 00000100:00080000:0.0:1789761523.203696:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761523.203699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761523.203707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761523.203715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048754432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761523.203737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761523.203741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761523.203743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761523.203747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 972 00000100:00080000:0.0:1789761523.203752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761523.203759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761523.203765:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048754304:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761523.203768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761523.203792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048754560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761523.203805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761523.203809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761523.203812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761523.203821:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 972 00000100:00080000:0.0:1789761523.203829:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761523.203832:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761523.203837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761523.203847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048754688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761523.203854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761523.203856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761523.203857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761523.203865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 972 00000800:00000100:3.0:1789761523.203865:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761523.203874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761523.203877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761523.203881:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761523.203890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048754816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761523.203934:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761523/real 1789761523] req@ffff9811c2e29180 x1876700048754304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761523.203963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048754304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761523.203979:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761523.203995:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761523.203999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048754304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761523.204007:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048754432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761523.204029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761523.204040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761523/real 1789761523] req@ffff9811c2e2aa00 x1876700048754432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761523.204049:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048754432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761523.204055:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761523.204057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761523.204060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048754432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761523.204083:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761523.204095:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761523.204102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761523/real 1789761523] req@ffff9811c2e2b100 x1876700048754560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761523.204109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048754560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761523.204115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761523.204117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761523.204119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761523.204124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761523.204143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761523.204150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761523/real 1789761523] req@ffff9811c2e29880 x1876700048754688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761523.204168:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048754688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761523.204174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761523.204176:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761523.204179:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761523.204183:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761523.204195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761523.204203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761523/real 1789761523] req@ffff9811c2e2ad80 x1876700048754816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761523.204216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048754816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761523.204224:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761523.204227:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761523.204231:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048754816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761523.459585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761524.099611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761524.483570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761525.123579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761525.507590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761526.147652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761526.531558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761527.171610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761527.555571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761528.195681:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761528.323578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761528.323589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761528.323593:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761528.323604:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 977 00000100:00080000:1.0:1789761528.323616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761528.323624:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761528.323683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048754944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761528.323720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761528.323723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761528.323728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761528.323732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 977 00000100:00080000:1.0:1789761528.323735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761528.323738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761528.323748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761528.323759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048755072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761528.323769:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048754944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761528.323771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761528.323778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761528.323780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761528.323784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 977 00000100:00080000:1.0:1789761528.323786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761528.323788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761528.323809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761528.323820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048755200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761528.323828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761528.323832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761528.323833:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761528.323835:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 977 00000100:00080000:1.0:1789761528.323838:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761528.323841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761528.323844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761528.323852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048755328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761528.323863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761528.323867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761528.323871:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761528.323878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 977 00000100:00080000:1.0:1789761528.323882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761528.323887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761528.323892:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048755072:192.168.203.113@tcp:38:kworker.0 00000008:00100000:1.0:1789761528.323897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761528.323917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048755456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761528.323937:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761528.323982:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761528.324009:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761528.579542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761529.219646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761529.603553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761530.243614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761530.627534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761531.267623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761531.395767:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789761531.395815:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761531.395831:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761531.395874:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761531.395882:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761531.395891:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761531.395900:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789761531.395935:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761528/real 1789761531] req@ffff9811c92c4000 x1876700048755456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761531.395963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048755456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761531.395976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761531.395982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761531.395986:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761531.395995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761528/real 1789761531] req@ffff9811c92c5500 x1876700048755072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761531.396008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048755072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761531.396016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761531.396019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761531.396023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048755072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761531.396029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761528/real 1789761531] req@ffff9811c92c5880 x1876700048755200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761531.396040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048755200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761531.396068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761531.396073:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761531.396076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761531.396083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761528/real 1789761531] req@ffff9811c92c4e00 x1876700048755328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761531.396095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048755328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761531.396103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761531.396106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761531.396109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761531.396115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761528/real 1789761531] req@ffff9811c92c4380 x1876700048754944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761531.396126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048754944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761531.396133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761531.396135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761531.396139:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048754944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761531.651555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761532.291619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761532.675540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761533.315690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761533.443585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761533.443593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761533.443597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761533.443606:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 983 00000100:00080000:1.0:1789761533.443616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761533.443622:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761533.443667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048755584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761533.443688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761533.443693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761533.443695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761533.443701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 983 00000100:00080000:1.0:1789761533.443706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761533.443712:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048755584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761533.443713:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761533.443729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761533.443749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048755712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761533.443762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761533.443765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761533.443767:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761533.443771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 983 00000100:00080000:1.0:1789761533.443776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761533.443781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789761533.443797:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761533.443810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761533.443827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048755840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761533.443839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761533.443844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761533.443850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761533.443850:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761533/real 1789761533] req@ffff9811c92c5f80 x1876700048755584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761533.443856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 983 00000100:00080000:1.0:1789761533.443863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761533.443872:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761533.443881:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761533.443890:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048755584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761533.443895:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048755968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761533.443914:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761533.443916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761533.443923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761533.443925:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761533.443929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 983 00000100:00080000:3.0:1789761533.443934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789761533.443935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761533.443944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761533.443944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048755584:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789761533.443954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761533.443968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048756096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761533.443982:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048755712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761533.444013:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761533.444031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761533/real 1789761533] req@ffff9811c92c7480 x1876700048755712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761533.444044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048755712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761533.444053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761533.444057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761533.444060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048755712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761533.444067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761533.444088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761533.444102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761533/real 1789761533] req@ffff9811c92c4a80 x1876700048755840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761533.444114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048755840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761533.444134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761533.444138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761533.444141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761533.444148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761533.444172:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761533.444184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761533/real 1789761533] req@ffff9811c92c7800 x1876700048755968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761533.444195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048755968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761533.444202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761533.444205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761533.444208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048755968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761533.444225:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761533.444252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761533.444270:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761533/real 1789761533] req@ffff9811c92c7100 x1876700048756096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761533.444284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048756096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761533.444291:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761533.444293:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761533.444300:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761533.699564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761534.339654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761534.723547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761535.363663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761535.747528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761536.387624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761536.771528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761537.411678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761537.795520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761538.435654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761538.563597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761538.563608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761538.563626:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761538.563638:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 988 00000100:00080000:3.0:1789761538.563650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761538.563657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761538.563702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048756224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761538.563728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761538.563731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761538.563733:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761538.563736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 988 00000100:00080000:3.0:1789761538.563738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761538.563741:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761538.563750:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761538.563758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048756352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761538.563767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761538.563770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761538.563772:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761538.563785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048756224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761538.563796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 988 00000100:00080000:3.0:1789761538.563800:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761538.563803:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761538.563807:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761538.563819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048756480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761538.563827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761538.563829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761538.563831:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761538.563834:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 988 00000100:00080000:3.0:1789761538.563851:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761538.563853:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761538.563868:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761538.563876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048756608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761538.563882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761538.563889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761538.563891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761538.563893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761538.563896:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 988 00000100:00080000:3.0:1789761538.563899:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761538.563901:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761538.563904:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761538.563909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048756736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761538.563962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761538/real 1789761538] req@ffff9811e1a22d80 x1876700048756224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761538.563982:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048756224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761538.563992:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761538.563997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761538.564001:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048756224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761538.564015:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048756352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761538.564040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761538.564053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761538/real 1789761538] req@ffff9811e1a22680 x1876700048756352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761538.564064:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048756352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761538.564072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761538.564076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761538.564079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048756352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761538.564105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761538.564123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761538.564133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761538/real 1789761538] req@ffff9811e1a21880 x1876700048756480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761538.564143:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048756480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761538.564151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761538.564154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761538.564157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761538.564163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761538.564179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761538.564187:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761538/real 1789761538] req@ffff9811e1a21c00 x1876700048756608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761538.564206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048756608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761538.564214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761538.564217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761538.564220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761538.564226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761538.564245:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761538.564256:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761538/real 1789761538] req@ffff9811e1a22a00 x1876700048756736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761538.564275:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048756736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761538.564283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761538.564286:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761538.564289:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048756736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761538.819511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761539.459635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761539.843517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761540.483658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761540.867621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761541.507667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761541.891553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761542.531592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761542.915593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761543.555648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761543.683546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761543.683558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761543.683563:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761543.683585:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 993 00000100:00080000:3.0:1789761543.683597:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761543.683606:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761543.683640:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048756864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761543.683673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761543.683678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761543.683680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761543.683686:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 993 00000100:00080000:3.0:1789761543.683692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761543.683696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761543.683707:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761543.683724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048756992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761543.683733:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048756864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761543.683741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761543.683746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761543.683749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761543.683790:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761543.683823:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761543/real 1789761543] req@ffff9811e1a21180 x1876700048756864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761543.683839:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 993 00000100:00100000:2.0:1789761543.683844:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048756864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761543.683847:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761543.683857:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761543.683870:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761543.683874:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761543.683875:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761543.683881:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048756864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761543.683895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048757120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761543.683903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048756992:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761543.683915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761543.683920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761543.683922:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761543.683927:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 993 00000100:00080000:3.0:1789761543.683932:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761543.683938:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761543.683954:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761543/real 1789761543] req@ffff9811e1a22300 x1876700048756992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761543.683969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048756992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761543.683978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761543.683982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761543.683986:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048756992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761543.683996:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761543.684015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761543.684025:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761543/real 1789761543] req@ffff9811e1a20700 x1876700048757120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761543.684033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048757120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761543.684039:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761543.684041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761543.684043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757120:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761543.684050:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761543.684059:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761543.684075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048757248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761543.684093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761543.684120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757248:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789761543.684139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761543.684142:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761543.684144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761543.684151:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 993 00000100:00080000:3.0:1789761543.684157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761543.684158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761543/real 1789761543] req@ffff9811e1a23100 x1876700048757248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761543.684165:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761543.684173:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048757248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761543.684175:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761543.684186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761543.684189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761543.684192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761543.684197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048757376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761543.684236:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761543.684251:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761543.684259:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761543/real 1789761543] req@ffff9811e1a23480 x1876700048757376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761543.684266:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048757376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761543.684271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761543.684273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761543.684275:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761543.939536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761544.579655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761544.963550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761545.603660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761545.987567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761546.627622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761547.011610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761547.651647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761548.035578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761548.675615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761548.803600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761548.803611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761548.803616:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761548.803628:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 998 00000100:00080000:3.0:1789761548.803640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761548.803648:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761548.803692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048757504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761548.803719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761548.803723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761548.803724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761548.803727:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 998 00000100:00080000:3.0:1789761548.803729:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761548.803731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761548.803739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761548.803750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048757632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761548.803759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761548.803762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761548.803763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761548.803765:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 998 00000100:00100000:2.0:1789761548.803784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048757504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761548.803794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761548.803803:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761548.803812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761548.803830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048757760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761548.803843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761548.803848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761548.803850:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761548.803855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 998 00000800:00000100:2.0:1789761548.803863:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761548.803865:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761548.803874:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761548.803882:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761548.803898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048757888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761548.803905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761548.803908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761548.803910:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761548.803914:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761548/real 1789761548] req@ffff9811e1a20000 x1876700048757504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761548.803920:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 998 00000100:00080000:3.0:1789761548.803923:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761548.803930:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761548.803946:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761548.803961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048757504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761548.803967:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048758016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761548.804007:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761548.804016:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761548.804025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048757504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761548.804047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048757632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761548.804075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761548.804091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761548/real 1789761548] req@ffff9811e1a23800 x1876700048757632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761548.804103:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048757632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761548.804111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761548.804115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761548.804120:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048757632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761548.804154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761548.804179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761548.804193:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761548/real 1789761548] req@ffff9811e1a21f80 x1876700048757760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761548.804206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048757760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761548.804222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761548.804225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761548.804227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761548.804232:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761548.804249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761548.804258:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761548/real 1789761548] req@ffff9811e1a20a80 x1876700048757888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761548.804265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048757888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761548.804271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761548.804273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761548.804275:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048757888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761548.804279:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761548.804289:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761548.804294:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761548/real 1789761548] req@ffff9811e1a20e00 x1876700048758016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761548.804301:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048758016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761548.804306:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761548.804308:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761548.804315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761549.059617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761549.699635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761550.083568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761550.723639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761551.107629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761551.747602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761552.131561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761552.771664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761553.155564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761553.795648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761553.923557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761553.923567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761553.923572:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761553.923582:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1003 00000100:00080000:1.0:1789761553.923594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761553.923601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761553.923645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048758144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761553.923681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761553.923685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761553.923691:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761553.923696:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1003 00000100:00080000:1.0:1789761553.923702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761553.923718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761553.923727:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048758144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789761553.923731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761553.923749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048758272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761553.923760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761553.923763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761553.923765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761553.923768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1003 00000100:00080000:1.0:1789761553.923785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761553.923789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761553.923794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761553.923803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048758400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761553.923812:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761553.923815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761553.923819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761553.923820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761553.923824:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1003 00000100:00080000:1.0:1789761553.923827:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761553.923829:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761553.923832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761553.923841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048758528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761553.923847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761553.923849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761553.923850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761553.923853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1003 00000100:00080000:1.0:1789761553.923855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761553.923858:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761553.923861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761553.923869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048758656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761553.923877:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761553/real 1789761553] req@ffff9811c92c6a00 x1876700048758144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761553.923908:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048758144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761553.923917:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761553.923921:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761553.923931:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048758144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761553.923954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048758272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761553.923973:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761553.923983:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761553/real 1789761553] req@ffff9811c92c4700 x1876700048758272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761553.923992:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048758272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761553.923998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761553.924000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761553.924002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048758272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761553.924006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761553.924017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761553.924032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761553/real 1789761553] req@ffff9811c92c6300 x1876700048758400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761553.924040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048758400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761553.924046:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761553.924048:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761553.924050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761553.924053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761553.924066:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761553.924072:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761553/real 1789761553] req@ffff9811c92c7100 x1876700048758528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761553.924089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048758528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761553.924098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761553.924100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761553.924103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761553.924106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761553.924118:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761553.924125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761553/real 1789761553] req@ffff9811c92c7800 x1876700048758656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761553.924132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048758656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761553.924143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761553.924145:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761553.924147:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048758656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761554.179574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761554.819633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761555.203582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761555.843613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761556.227553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761556.867613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761557.251555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761557.891605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761558.275574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761558.915639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761559.043596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761559.043608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761559.043612:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761559.043623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1008 00000100:00080000:0.0:1789761559.043635:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761559.043643:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761559.043683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048758784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761559.043706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761559.043710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761559.043711:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761559.043714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1008 00000100:00080000:0.0:1789761559.043717:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761559.043719:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761559.043729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761559.043739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048758912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761559.043750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048758784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761559.043751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761559.043759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761559.043778:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761559.043790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1008 00000100:00080000:0.0:1789761559.043794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761559.043798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761559.043803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761559.043815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048759040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761559.043823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761559.043826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761559.043828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761559.043833:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761559.043834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1008 00000100:00080000:0.0:1789761559.043840:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761559.043843:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761559.043847:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761559.043856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048759168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761559.043863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761559.043866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761559.043868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761559.043871:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1008 00000100:00080000:0.0:1789761559.043874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761559.043876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761559.043880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761559.043882:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761559/real 1789761559] req@ffff9811c2e29c00 x1876700048758784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761559.043900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048759296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761559.043906:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048758784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761559.043920:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761559.043924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761559.043929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048758784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761559.043964:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048758912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761559.043981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761559.043991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761559/real 1789761559] req@ffff9811c2e2a680 x1876700048758912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761559.044000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048758912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761559.044006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761559.044009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761559.044011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048758912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761559.044022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761559.044037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761559.044045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761559/real 1789761559] req@ffff9811c2e28700 x1876700048759040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761559.044067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048759040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761559.044076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761559.044081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761559.044084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761559.044092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761559.044114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761559.044125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761559/real 1789761559] req@ffff9811c2e2ad80 x1876700048759168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761559.044138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048759168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761559.044147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761559.044151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761559.044154:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761559.044161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761559.044182:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761559.044194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761559/real 1789761559] req@ffff9811c2e29880 x1876700048759296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761559.044209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048759296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761559.044215:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761559.044217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761559.044219:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761559.299598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761559.939607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761560.323587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761560.963624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761561.347602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761561.987590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761562.371607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761563.011636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761563.395552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761564.035613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761564.163618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761564.163632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761564.163637:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761564.163648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1013 00000100:00080000:3.0:1789761564.163661:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761564.163668:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761564.163711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048759424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761564.163737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761564.163740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761564.163741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761564.163744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1013 00000100:00080000:3.0:1789761564.163746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761564.163749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761564.163758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761564.163784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048759424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761564.163799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048759552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761564.163813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761564.163818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761564.163820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761564.163825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1013 00000100:00080000:3.0:1789761564.163842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761564.163849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761564.163854:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761564.163858:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761564.163869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048759680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761564.163878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761564.163881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761564.163883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761564.163886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1013 00000100:00080000:3.0:1789761564.163889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761564.163891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761564.163895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761564.163904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048759808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761564.163909:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761564/real 1789761564] req@ffff9811e1a20380 x1876700048759424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761564.163915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761564.163923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761564.163926:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761564.163938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1013 00000100:00100000:2.0:1789761564.163940:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048759424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761564.163946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761564.163953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761564.163955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761564.163960:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761564.163965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048759424:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761564.163966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761564.163982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048759936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761564.163990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048759552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761564.164011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761564.164026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761564/real 1789761564] req@ffff9811e1a21500 x1876700048759552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761564.164057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048759552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761564.164067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761564.164071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761564.164075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048759552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761564.164082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761564.164107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761564.164121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761564/real 1789761564] req@ffff9811e1a20e00 x1876700048759680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761564.164136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048759680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761564.164144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761564.164148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761564.164152:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761564.164160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761564.164182:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761564.164191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761564/real 1789761564] req@ffff9811e1a20a80 x1876700048759808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761564.164200:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048759808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761564.164215:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761564.164218:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761564.164221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761564.164233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761564.164248:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761564.164255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761564/real 1789761564] req@ffff9811e1a21f80 x1876700048759936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761564.164263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048759936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761564.164269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761564.164271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761564.164273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048759936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761564.419598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761565.059726:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761565.443576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761566.083592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761566.467589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761567.107655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761567.491571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761568.131580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761568.515579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761569.155655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761569.283554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761569.283566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761569.283571:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761569.283583:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1018 00000100:00080000:0.0:1789761569.283594:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761569.283602:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761569.283646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048760064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761569.283676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761569.283681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761569.283684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761569.283688:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1018 00000100:00080000:0.0:1789761569.283692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761569.283716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761569.283728:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761569.283734:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048760064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761569.283745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048760192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761569.283756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761569.283759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761569.283761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761569.283764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1018 00000100:00080000:0.0:1789761569.283767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761569.283770:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761569.283774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761569.283784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048760320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761569.283791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761569.283794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761569.283795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761569.283797:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1018 00000100:00080000:0.0:1789761569.283800:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761569.283802:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761569.283805:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761569.283812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048760448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761569.283818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761569.283826:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761569.283827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761569.283831:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761569.283836:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1018 00000100:00080000:0.0:1789761569.283840:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761569.283843:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761569.283847:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761569.283856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048760576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761569.283887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761569/real 1789761569] req@ffff9811c2e2b100 x1876700048760064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761569.283904:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048760064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761569.283915:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761569.283930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761569.283935:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048760064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761569.283964:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048760192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761569.283988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761569.283998:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761569/real 1789761569] req@ffff9811c2e2aa00 x1876700048760192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761569.284008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048760192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761569.284014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761569.284017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761569.284019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048760192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761569.284023:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761569.284034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761569.284041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761569/real 1789761569] req@ffff9811c2e29180 x1876700048760320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761569.284049:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048760320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761569.284054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761569.284056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761569.284058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761569.284061:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761569.284082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761569.284088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761569/real 1789761569] req@ffff9811c2e2bb80 x1876700048760448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761569.284096:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048760448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761569.284102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761569.284103:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761569.284105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761569.284109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761569.284132:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761569.284138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761569/real 1789761569] req@ffff9811c2e28e00 x1876700048760576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761569.284146:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048760576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761569.284151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761569.284153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761569.284155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761569.539551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761570.179652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761570.563569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761571.203613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761571.587563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761572.227613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761572.611602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761573.251616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761573.635638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761574.275594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761574.403545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761574.403552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761574.403555:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761574.403564:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1023 00000100:00080000:0.0:1789761574.403573:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761574.403579:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761574.403609:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048760704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761574.403648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761574.403652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761574.403654:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761574.403658:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1023 00000100:00080000:0.0:1789761574.403662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761574.403667:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761574.403677:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761574.403690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048760832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761574.403696:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048760704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761574.403702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761574.403705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761574.403707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761574.403710:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1023 00000100:00080000:0.0:1789761574.403713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761574.403717:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761574.403721:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761574.403730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048760960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761574.403747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761574.403750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761574.403751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761574.403755:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1023 00000800:00000100:2.0:1789761574.403756:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761574.403765:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761574.403769:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761574.403774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761574.403784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048761088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761574.403794:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761574/real 1789761574] req@ffff9811c2e28380 x1876700048760704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761574.403802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761574.403806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761574.403807:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761574.403811:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1023 00000100:00080000:0.0:1789761574.403814:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761574.403818:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761574.403822:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761574.403827:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048760704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761574.403831:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048761216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761574.403839:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761574.403842:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761574.403846:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048760704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761574.403879:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048760832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761574.403899:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761574.403910:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761574/real 1789761574] req@ffff9811c2e28a80 x1876700048760832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761574.403918:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048760832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761574.403924:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761574.403926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761574.403928:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048760832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761574.403932:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761574.403942:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761574.403948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761574/real 1789761574] req@ffff9811c2e29f80 x1876700048760960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761574.403955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048760960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761574.403970:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761574.403973:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761574.403975:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048760960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761574.403979:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761574.403993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761574.403999:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761574/real 1789761574] req@ffff9811c2e2b800 x1876700048761088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761574.404007:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048761088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761574.404012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761574.404013:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761574.404015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761574.404018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761574.404028:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761574.404034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761574/real 1789761574] req@ffff9811c2e2b480 x1876700048761216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761574.404041:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048761216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761574.404046:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761574.404047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761574.404054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761574.659554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761575.299692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761575.683579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761576.323668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761576.707567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761577.347617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761577.731534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761578.371654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761578.755564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761579.395616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761579.523583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761579.523597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761579.523621:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761579.523635:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1029 00000100:00080000:0.0:1789761579.523649:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761579.523659:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761579.523703:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048761344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761579.523730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761579.523735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761579.523738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761579.523743:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1029 00000100:00080000:0.0:1789761579.523747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761579.523752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761579.523764:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761579.523768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048761344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761579.523781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048761472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761579.523795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761579.523799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761579.523801:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761579.523805:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1029 00000100:00080000:0.0:1789761579.523809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761579.523813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761579.523818:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761579.523827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048761600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761579.523842:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761579.523848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761579.523852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761579.523854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761579.523858:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1029 00000100:00080000:0.0:1789761579.523862:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761579.523866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761579.523880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761579.523894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048761728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761579.523903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761579.523906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761579.523908:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761579.523912:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1029 00000100:00080000:0.0:1789761579.523915:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761579.523921:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761579.523926:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761579.523937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048761856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761579.524142:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761579/real 1789761579] req@ffff9811c2e29500 x1876700048761344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761579.524165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048761344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761579.524176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761579.524181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761579.524185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048761344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761579.524211:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048761472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761579.524231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761579.524242:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761579/real 1789761579] req@ffff9811c2e28000 x1876700048761472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761579.524251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048761472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761579.524267:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761579.524269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761579.524272:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048761472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761579.524276:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761579.524290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761579.524297:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761579/real 1789761579] req@ffff9811c2e2a300 x1876700048761600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761579.524321:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048761600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761579.524327:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761579.524330:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761579.524332:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761579.524336:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761579.524352:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761579.524359:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761579/real 1789761579] req@ffff9811c2e2b480 x1876700048761728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761579.524366:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048761728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761579.524378:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761579.524380:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761579.524387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761579.524392:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761579.524407:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761579.524415:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761579/real 1789761579] req@ffff9811c2e2b800 x1876700048761856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761579.524422:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048761856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761579.524427:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761579.524429:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761579.524431:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048761856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761579.779602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761580.419613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761580.803563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761581.443627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761581.827618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761582.467638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761582.851538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761583.491605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761583.875614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761584.515700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761584.643541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761584.643554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761584.643559:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761584.643571:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1034 00000100:00080000:1.0:1789761584.643582:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761584.643590:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761584.643633:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048761984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761584.643663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761584.643666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761584.643668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761584.643671:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1034 00000100:00080000:1.0:1789761584.643674:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761584.643677:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761584.643687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761584.643700:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048761984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761584.643701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048762112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761584.643721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761584.643724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761584.643726:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761584.643729:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1034 00000100:00080000:1.0:1789761584.643731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761584.643734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761584.643755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761584.643764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048762240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761584.643773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761584.643775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761584.643777:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761584.643779:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1034 00000100:00080000:1.0:1789761584.643782:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761584.643784:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761584.643788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761584.643791:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761584.643803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048762368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761584.643810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761584.643814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761584.643815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761584.643818:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1034 00000100:00080000:1.0:1789761584.643827:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761584.643830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761584.643833:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761584.643841:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048762496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761584.643853:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761584/real 1789761584] req@ffff9811c92c4a80 x1876700048761984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761584.643871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048761984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761584.643880:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761584.643883:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761584.643886:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048761984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761584.643918:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048762112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761584.643952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761584.643967:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761584/real 1789761584] req@ffff9811c92c7480 x1876700048762112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761584.643982:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048762112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761584.643990:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761584.643993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761584.643995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048762112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761584.644000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761584.644016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761584.644024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761584/real 1789761584] req@ffff9811c92c5f80 x1876700048762240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761584.644032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048762240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761584.644037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761584.644039:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761584.644041:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761584.644045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761584.644056:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761584.644062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761584/real 1789761584] req@ffff9811c92c4380 x1876700048762368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761584.644080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048762368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761584.644086:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761584.644088:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761584.644090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761584.644094:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761584.644106:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761584.644112:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761584/real 1789761584] req@ffff9811c92c4e00 x1876700048762496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761584.644120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048762496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761584.644125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761584.644126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761584.644128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761584.899551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761585.539575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761585.923552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761586.563598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761586.947526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761587.587612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761587.971562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761588.611648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761588.995510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761589.635661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761589.763591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761589.763602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761589.763607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761589.763617:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1039 00000100:00080000:2.0:1789761589.763629:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761589.763637:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761589.763672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048762624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761589.763701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761589.763706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761589.763708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761589.763713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1039 00000100:00080000:2.0:1789761589.763717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761589.763722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761589.763732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761589.763744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048762752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761589.763753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761589.763755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761589.763756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761589.763759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1039 00000100:00080000:2.0:1789761589.763763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761589.763783:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048762624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761589.763786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761589.763801:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761589.763829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048762880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761589.763842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761589.763846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761589.763848:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761589.763853:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1039 00000100:00080000:2.0:1789761589.763869:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761589.763875:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761589.763882:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789761589.763889:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761589.763898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048763008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761589.763913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761589.763917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761589.763919:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761589.763924:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1039 00000100:00080000:2.0:1789761589.763929:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761589.763936:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761589.763942:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761589.763954:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048763136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761589.763965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761589/real 1789761589] req@ffff9811d8639880 x1876700048762624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761589.763988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048762624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761589.764014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761589.764018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761589.764021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048762624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761589.764028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048762752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761589.764050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761589.764061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761589/real 1789761589] req@ffff9811d8639500 x1876700048762752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761589.764069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048762752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761589.764075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761589.764078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761589.764080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048762752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761589.764107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761589.764129:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761589.764142:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761589/real 1789761589] req@ffff9811d8639f80 x1876700048762880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761589.764155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048762880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761589.764164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761589.764167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761589.764171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048762880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761589.764190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761589.764210:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761589.764219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761589/real 1789761589] req@ffff9811d863a680 x1876700048763008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761589.764242:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048763008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761589.764249:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761589.764251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761589.764253:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761589.764258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761589.764274:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761589.764282:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761589/real 1789761589] req@ffff9811d8638a80 x1876700048763136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761589.764289:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048763136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761589.764294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761589.764296:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761589.764298:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761590.019551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761590.659669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761591.043577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761591.683658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761592.067587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761592.707632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761593.091548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761593.731617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761594.115587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761594.755679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761594.883587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761594.883598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761594.883602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761594.883634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1044 00000100:00080000:3.0:1789761594.883645:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761594.883652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761594.883685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048763264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761594.883718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761594.883722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761594.883724:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761594.883729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1044 00000100:00080000:3.0:1789761594.883733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761594.883737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761594.883747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761594.883758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048763264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761594.883763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048763392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761594.883780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761594.883784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761594.883786:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761594.883790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1044 00000100:00080000:3.0:1789761594.883793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761594.883797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761594.883803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761594.883815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048763520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761594.883840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761594.883844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761594.883846:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761594.883851:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1044 00000100:00080000:3.0:1789761594.883856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761594.883861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761594.883868:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048763392:192.168.203.113@tcp:38:kworker.0 00000008:00100000:3.0:1789761594.883869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761594.883892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048763648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761594.883904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761594.883906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763520:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761594.883913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761594.883917:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761594.883923:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1044 00000100:00080000:3.0:1789761594.883928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761594.883932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761594.883939:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761594.883946:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761594.883956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048763776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761594.883986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761595.139534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761595.779625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761596.163566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761596.803625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761597.187567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761597.827630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761597.955751:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789761597.955798:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761597.955815:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761597.955853:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761597.955883:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761597.955895:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761597.955912:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761597.955936:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761594/real 1789761597] req@ffff9811e1a23100 x1876700048763648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761597.955978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048763648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761597.956004:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761597.956010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761597.956015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761597.956025:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761594/real 1789761597] req@ffff9811e1a20000 x1876700048763392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761597.956043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048763392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761597.956053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761597.956057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761597.956060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048763392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761597.956090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761594/real 1789761597] req@ffff9811e1a23480 x1876700048763520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761597.956106:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048763520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761597.956115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761597.956119:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761597.956122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761597.956129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761594/real 1789761597] req@ffff9811e1a23800 x1876700048763264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761597.956142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048763264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761597.956149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761597.956153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761597.956156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048763264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761597.956206:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761594/real 1789761597] req@ffff9811e1a20700 x1876700048763776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761597.956221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048763776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761597.956230:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761597.956234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761597.956238:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048763776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761598.211544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761598.851650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761599.235527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761599.875627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761600.003575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761600.003586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761600.003605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761600.003616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1049 00000100:00080000:1.0:1789761600.003636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761600.003644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761600.003689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048763904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761600.003723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761600.003727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761600.003729:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761600.003734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1049 00000100:00080000:1.0:1789761600.003739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761600.003743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761600.003756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761600.003773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048764032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761600.003785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048763904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761600.003788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761600.003796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761600.003798:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761600.003803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1049 00000100:00080000:1.0:1789761600.003807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761600.003818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761600.003825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761600.003839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048764160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761600.003864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761600.003868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761600.003870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761600.003878:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761600.003888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1049 00000100:00080000:1.0:1789761600.003895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761600.003901:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761600.003907:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761600.003924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048764288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761600.003936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761600.003937:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761600/real 1789761600] req@ffff9811c92c5880 x1876700048763904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761600.003944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761600.003948:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761600.003958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1049 00000100:00080000:1.0:1789761600.003963:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761600.003969:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761600.003973:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048763904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761600.003976:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761600.003993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761600.003996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048764416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761600.004001:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761600.004011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048763904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761600.004047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048764032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761600.004084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761600.004100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761600/real 1789761600] req@ffff9811c92c5500 x1876700048764032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761600.004112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048764032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761600.004121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761600.004124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761600.004128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048764032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761600.004160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761600.004186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761600.004210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761600/real 1789761600] req@ffff9811c92c4000 x1876700048764160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761600.004224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048764160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761600.004232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761600.004235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761600.004239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761600.004245:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761600.004269:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761600.004281:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761600/real 1789761600] req@ffff9811c92c6680 x1876700048764288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761600.004293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048764288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761600.004301:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761600.004305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761600.004308:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761600.004314:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761600.004333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761600.004343:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761600/real 1789761600] req@ffff9811c92c5c00 x1876700048764416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761600.004355:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048764416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761600.004371:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761600.004374:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761600.004377:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761600.259596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761600.899700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761601.283528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761601.923641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761602.307517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761602.947617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761603.331531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761603.971601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761604.355585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761604.995661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761605.123587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761605.123595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761605.123598:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761605.123607:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1054 00000100:00080000:3.0:1789761605.123630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761605.123637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761605.123664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048764544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761605.123691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761605.123694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761605.123695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761605.123698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1054 00000100:00080000:3.0:1789761605.123700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761605.123702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761605.123709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761605.123718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048764672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761605.123727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761605.123729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761605.123731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761605.123734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1054 00000100:00080000:3.0:1789761605.123736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761605.123741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761605.123743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048764544:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761605.123745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761605.123755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048764800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761605.123772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761605.123779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761605.123781:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761605.123785:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1054 00000100:00080000:3.0:1789761605.123793:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761605.123798:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789761605.123803:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789761605.123807:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761605.123824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048764928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761605.123835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761605.123839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789761605.123842:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761605/real 1789761605] req@ffff9811e1a22300 x1876700048764544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761605.123843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761605.123850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1054 00000100:00080000:3.0:1789761605.123858:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761605.123864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761605.123870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761605.123871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048764544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761605.123886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048765056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761605.123892:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761605.123898:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761605.123902:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048764544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761605.123908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048764672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761605.123926:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761605.123934:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761605/real 1789761605] req@ffff9811e1a21180 x1876700048764672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761605.123942:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048764672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761605.123948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761605.123958:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761605.123960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048764672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761605.123979:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761605.123994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761605.124001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761605/real 1789761605] req@ffff9811e1a22a00 x1876700048764800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761605.124009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048764800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761605.124014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761605.124016:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761605.124018:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761605.124021:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761605.124031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761605.124036:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761605/real 1789761605] req@ffff9811e1a21c00 x1876700048764928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761605.124043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048764928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761605.124048:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761605.124050:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761605.124052:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048764928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761605.124055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761605.124071:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761605.124077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761605/real 1789761605] req@ffff9811e1a21880 x1876700048765056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761605.124085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048765056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761605.124090:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761605.124092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761605.124094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761605.379540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761606.019602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761606.403518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761607.043609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761607.427612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761608.067588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761608.451543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761609.091621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761609.475581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761610.115604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761610.243597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761610.243608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761610.243620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761610.243635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1059 00000100:00080000:1.0:1789761610.243670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761610.243678:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761610.243721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048765184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761610.243753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761610.243757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761610.243758:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761610.243762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1059 00000100:00080000:1.0:1789761610.243765:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761610.243768:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761610.243778:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761610.243790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048765312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761610.243800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761610.243803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761610.243804:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761610.243807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1059 00000100:00080000:1.0:1789761610.243811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761610.243813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761610.243817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761610.243828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048765440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761610.243832:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048765184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761610.243837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761610.243846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761610.243848:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761610.243851:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1059 00000100:00080000:1.0:1789761610.243854:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761610.243859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761610.243865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761610.243877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048765568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761610.243886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761610.243888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761610.243890:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761610.243893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1059 00000100:00080000:1.0:1789761610.243896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761610.243899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761610.243903:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761610.243912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048765696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761610.243921:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761610.243975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761610/real 1789761610] req@ffff9811c92c7b80 x1876700048765184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761610.244002:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048765184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761610.244013:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761610.244016:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761610.244019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048765184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761610.244027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048765312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761610.244050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761610.244092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761610/real 1789761610] req@ffff9811c92c5180 x1876700048765312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761610.244108:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048765312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761610.244118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761610.244122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761610.244126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048765312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761610.244157:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761610.244176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761610.244184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761610/real 1789761610] req@ffff9811c92c6d80 x1876700048765440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761610.244193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048765440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761610.244198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761610.244200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761610.244202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761610.244206:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761610.244217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761610.244223:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761610/real 1789761610] req@ffff9811c92c5c00 x1876700048765568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761610.244230:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048765568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761610.244242:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761610.244244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761610.244246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761610.244251:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761610.244264:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761610.244271:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761610/real 1789761610] req@ffff9811c92c6680 x1876700048765696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761610.244278:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048765696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761610.244283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761610.244285:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761610.244287:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048765696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761610.499544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761611.139598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761611.523574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761612.163613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761612.547541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761613.187582:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761613.571566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761614.211580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761614.595613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761615.235581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761615.363584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761615.363600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761615.363606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761615.363620:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1064 00000100:00080000:1.0:1789761615.363634:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761615.363643:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761615.363689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048765824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761615.363739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761615.363743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761615.363747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761615.363753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1064 00000100:00080000:1.0:1789761615.363767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761615.363773:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048765824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761615.363779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761615.363795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761615.363813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048765952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761615.363827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761615.363831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761615.363834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761615.363838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1064 00000100:00080000:1.0:1789761615.363843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761615.363848:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761615.363850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761615.363857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761615.363871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048766080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761615.363881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761615.363884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761615.363886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761615.363891:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1064 00000100:00100000:3.0:1789761615.363894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761615/real 1789761615] req@ffff9811c92c4000 x1876700048765824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761615.363904:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761615.363911:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761615.363916:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761615.363927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048766208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761615.363936:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761615.363939:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761615.363941:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761615.363945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1064 00000100:00080000:1.0:1789761615.363948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761615.363952:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761615.363957:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761615.363967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048766336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761615.364030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048765824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761615.364195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761615.364200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761615.364203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048765824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761615.364228:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048765952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761615.364286:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761615.364305:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761615/real 1789761615] req@ffff9811c92c5500 x1876700048765952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761615.364319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048765952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761615.364325:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761615.364328:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761615.364331:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048765952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761615.364336:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761615.364349:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761615.364357:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761615/real 1789761615] req@ffff9811c92c5880 x1876700048766080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761615.364365:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048766080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761615.364370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761615.364372:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761615.364373:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761615.364383:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761615.364398:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761615.364409:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761615/real 1789761615] req@ffff9811c92c4e00 x1876700048766208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761615.364422:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048766208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761615.364431:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761615.364434:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761615.364438:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761615.364445:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761615.364490:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761615.364507:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761615/real 1789761615] req@ffff9811c92c4380 x1876700048766336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761615.364523:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048766336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761615.364533:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761615.364537:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761615.364539:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761615.619555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761616.259580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761616.643573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761617.283613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761617.667568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761618.307617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761618.691523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761619.331649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761619.715533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761620.355580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761620.483559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761620.483574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761620.483578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761620.483589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1070 00000100:00080000:1.0:1789761620.483604:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761620.483614:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761620.483665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048766464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761620.483691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761620.483711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761620.483713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761620.483717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1070 00000100:00080000:1.0:1789761620.483721:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761620.483726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048766464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761620.483728:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761620.483744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761620.483764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048766592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761620.483779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761620.483806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761620.483808:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761620.483814:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761620.483835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1070 00000100:00080000:1.0:1789761620.483842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761620.483846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761620.483851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761620.483866:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761620/real 1789761620] req@ffff9811c92c5f80 x1876700048766464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761620.483867:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048766720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761620.483877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761620.483885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761620.483889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761620.483894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1070 00000100:00080000:1.0:1789761620.483897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761620.483900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761620.483903:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048766464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761620.483915:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789761620.483918:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761620.483923:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761620.483927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048766848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761620.483928:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048766464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761620.483945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761620.483948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761620.483950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761620.483953:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1070 00000100:00080000:1.0:1789761620.483957:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761620.483959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048766592:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761620.483964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761620.483969:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761620.483980:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048766976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761620.483988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761620.484019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761620/real 1789761620] req@ffff9811c92c7480 x1876700048766592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761620.484030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048766592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761620.484036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761620.484039:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761620.484043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048766592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761620.484050:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761620.484071:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761620.484084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761620/real 1789761620] req@ffff9811c92c4a80 x1876700048766720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761620.484098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048766720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761620.484116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761620.484118:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761620.484121:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761620.484125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761620.484143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761620.484150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761620/real 1789761620] req@ffff9811c92c7800 x1876700048766848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761620.484157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048766848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761620.484163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761620.484165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761620.484167:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761620.484179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761620.484190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761620.484196:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761620/real 1789761620] req@ffff9811c92c7100 x1876700048766976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761620.484203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048766976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761620.484208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761620.484210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761620.484216:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048766976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761620.739626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761621.379595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761621.763542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761622.403572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761622.787555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761623.427644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761623.811555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761624.451588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761624.835584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761625.475636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761625.603608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761625.603618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761625.603622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761625.603633:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1075 00000100:00080000:2.0:1789761625.603644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761625.603651:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761625.603688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048767104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761625.603718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761625.603720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761625.603722:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761625.603725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1075 00000100:00080000:2.0:1789761625.603728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761625.603731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761625.603741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761625.603753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048767232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761625.603766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761625.603771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761625.603773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761625.603784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1075 00000100:00100000:3.0:1789761625.603784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048767104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761625.603791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761625.603814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761625.603821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761625.603838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048767360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761625.603848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761625.603852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761625.603854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761625.603860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1075 00000100:00080000:2.0:1789761625.603874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761625.603879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761625.603885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761625.603900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048767488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761625.603900:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761625.603920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761625.603922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761625.603925:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761625.603929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1075 00000100:00080000:2.0:1789761625.603933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761625.603936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761625.603939:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761625.603947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048767616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761625.603996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761625/real 1789761625] req@ffff9811d863aa00 x1876700048767104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761625.604022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048767104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761625.604035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761625.604041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761625.604045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048767104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761625.604076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048767232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761625.604098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761625.604111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761625/real 1789761625] req@ffff9811d8638000 x1876700048767232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761625.604124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048767232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761625.604133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761625.604137:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761625.604141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048767232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761625.604147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761625.604168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761625.604194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761625/real 1789761625] req@ffff9811d863b800 x1876700048767360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761625.604203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048767360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761625.604209:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761625.604211:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761625.604213:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761625.604218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761625.604234:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761625.604240:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761625/real 1789761625] req@ffff9811d863b480 x1876700048767488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761625.604248:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048767488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761625.604253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761625.604255:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761625.604257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761625.604260:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761625.604270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761625.604276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761625/real 1789761625] req@ffff9811d8638380 x1876700048767616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761625.604282:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048767616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761625.604293:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761625.604295:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761625.604298:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048767616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761625.859544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761626.499643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761626.883628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761627.523567:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761627.907570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761628.547658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761628.931588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761629.571616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761629.955568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761630.595655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761630.723554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761630.723563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761630.723567:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761630.723574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1080 00000100:00080000:3.0:1789761630.723583:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761630.723589:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761630.723620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048767744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761630.723645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761630.723648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761630.723649:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761630.723652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1080 00000100:00080000:3.0:1789761630.723654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761630.723657:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761630.723664:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761630.723674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048767872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761630.723686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761630.723693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789761630.723696:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048767744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761630.723708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761630.723713:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1080 00000100:00080000:3.0:1789761630.723715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761630.723718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761630.723722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761630.723734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048768000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761630.723742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761630.723744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761630.723746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761630.723749:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1080 00000100:00080000:3.0:1789761630.723766:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761630.723769:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761630.723772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761630.723785:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761630.723787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048768128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761630.723802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761630.723806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761630.723809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761630.723815:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1080 00000100:00080000:3.0:1789761630.723820:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761630.723826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761630.723834:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761630.723835:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761630/real 1789761630] req@ffff9811e1a22680 x1876700048767744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761630.723861:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048767744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761630.723871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048768256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761630.723877:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761630.723890:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761630.723898:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048767744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761630.723943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048767872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761630.723974:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761630.723989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761630/real 1789761630] req@ffff9811e1a22d80 x1876700048767872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761630.724004:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048767872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761630.724014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761630.724018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761630.724022:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048767872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761630.724027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761630.724046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761630.724055:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761630/real 1789761630] req@ffff9811e1a21880 x1876700048768000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761630.724076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048768000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761630.724082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761630.724084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761630.724086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761630.724090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761630.724106:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761630.724113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761630/real 1789761630] req@ffff9811e1a21c00 x1876700048768128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761630.724121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048768128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761630.724126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761630.724128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761630.724130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761630.724133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761630.724143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761630.724149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761630/real 1789761630] req@ffff9811e1a22a00 x1876700048768256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761630.724156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048768256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761630.724171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761630.724173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761630.724176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761630.979564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761631.619679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761632.003576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761632.643634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761633.027550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761633.667557:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761634.051537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761634.691598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761635.075577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761635.715561:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761635.843545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761635.843554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761635.843557:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761635.843566:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1085 00000100:00080000:1.0:1789761635.843575:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761635.843581:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761635.843611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048768384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761635.843647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761635.843650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761635.843651:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761635.843655:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1085 00000100:00080000:1.0:1789761635.843658:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761635.843662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761635.843670:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761635.843678:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048768384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761635.843690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048768512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761635.843705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761635.843709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761635.843712:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761635.843716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1085 00000100:00080000:1.0:1789761635.843721:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761635.843726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761635.843732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761635.843737:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761635.843749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048768640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761635.843762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761635.843766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761635.843768:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761635.843770:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761635/real 1789761635] req@ffff9811c92c6300 x1876700048768384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761635.843775:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1085 00000100:00080000:1.0:1789761635.843788:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761635.843790:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048768384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761635.843801:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761635.843805:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761635.843807:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761635.843813:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048768384:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789761635.843817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761635.843839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048768768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761635.843845:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048768512:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761635.843861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761635.843865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761635.843867:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761635.843871:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761635.843872:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1085 00000100:00080000:1.0:1789761635.843880:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761635.843898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761635.843902:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761635/real 1789761635] req@ffff9811c92c4700 x1876700048768512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761635.843907:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761635.843916:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048768512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761635.843926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048768896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761635.843958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761635.843963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761635.843969:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048768512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761635.843977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761635.844001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761635.844010:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761635/real 1789761635] req@ffff9811c92c6a00 x1876700048768640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761635.844018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048768640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761635.844024:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761635.844026:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761635.844028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761635.844039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761635.844055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761635.844067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761635/real 1789761635] req@ffff9811c92c7100 x1876700048768768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761635.844079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048768768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761635.844088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761635.844091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761635.844105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761635.844112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761635.844136:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761635.844149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761635/real 1789761635] req@ffff9811c92c7800 x1876700048768896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761635.844162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048768896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761635.844171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761635.844184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761635.844189:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048768896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761636.099599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761636.739630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761637.123602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761637.763590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761638.147619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761638.787633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761639.171579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761639.811581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761640.195587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761640.835613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761640.963539:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761640.963547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761640.963550:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761640.963558:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1090 00000100:00080000:3.0:1789761640.963568:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761640.963576:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761640.963614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048769024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761640.963648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761640.963653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761640.963655:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761640.963658:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1090 00000100:00080000:3.0:1789761640.963661:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761640.963665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761640.963674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761640.963686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048769152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761640.963693:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048769024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761640.963697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761640.963703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761640.963705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761640.963708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1090 00000100:00080000:3.0:1789761640.963724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761640.963727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761640.963731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761640.963741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048769280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761640.963747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761640.963750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761640.963752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761640.963755:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1090 00000100:00080000:3.0:1789761640.963757:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761640.963759:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761640.963762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761640.963769:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761640.963771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048769408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761640.963778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761640.963781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761640.963782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761640.963785:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1090 00000100:00080000:3.0:1789761640.963787:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761640.963790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761640.963793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761640.963799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048769536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761640.963821:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761640/real 1789761640] req@ffff9811e1a21180 x1876700048769024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761640.963855:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048769024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761640.963868:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761640.963873:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761640.963878:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048769024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761640.963903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048769152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761640.963937:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761640.963948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761640/real 1789761640] req@ffff9811e1a22300 x1876700048769152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761640.963958:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048769152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761640.963964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761640.963966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761640.963969:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048769152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761640.963973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761640.963987:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761640.963994:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761640/real 1789761640] req@ffff9811e1a20700 x1876700048769280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761640.964001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048769280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761640.964006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761640.964009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761640.964011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761640.964014:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761640.964025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761640.964030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761640/real 1789761640] req@ffff9811e1a23800 x1876700048769408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761640.964049:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048769408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761640.964054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761640.964056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761640.964058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761640.964061:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761640.964073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761640.964079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761640/real 1789761640] req@ffff9811e1a23480 x1876700048769536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761640.964086:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048769536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761640.964091:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761640.964093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761640.964095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761641.219531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761641.859604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761642.243637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761642.883625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761643.267565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761643.907589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761644.291605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761644.931614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761645.315542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761645.955586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761646.083574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761646.083585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761646.083590:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761646.083600:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1095 00000100:00080000:1.0:1789761646.083611:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761646.083618:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761646.083667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048769664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761646.083717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761646.083721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761646.083723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761646.083729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1095 00000100:00080000:1.0:1789761646.083734:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761646.083740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761646.083752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761646.083756:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048769664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761646.083771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048769792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761646.083785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761646.083790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761646.083792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761646.083797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1095 00000100:00080000:1.0:1789761646.083802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761646.083819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761646.083823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761646.083834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048769920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761646.083844:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761646.083849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761646.083853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761646.083855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761646.083859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1095 00000100:00080000:1.0:1789761646.083869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761646.083873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761646.083878:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761646.083892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048770048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761646.083892:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761646/real 1789761646] req@ffff9811c92c4a80 x1876700048769664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761646.083923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761646.083926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761646.083927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761646.083934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1095 00000100:00080000:1.0:1789761646.083940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761646.083945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761646.083949:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761646.083955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048769664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761646.083959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048770176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761646.083976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761646.083982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761646.083986:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048769664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761646.084010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048769792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761646.084029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761646.084038:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761646/real 1789761646] req@ffff9811c92c7480 x1876700048769792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761646.084046:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048769792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761646.084052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761646.084054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761646.084057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048769792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761646.084060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761646.084071:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761646.084077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761646/real 1789761646] req@ffff9811c92c5f80 x1876700048769920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761646.084085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048769920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761646.084100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761646.084104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761646.084107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048769920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761646.084114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761646.084137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761646.084149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761646/real 1789761646] req@ffff9811c92c4380 x1876700048770048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761646.084162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048770048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761646.084168:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761646.084170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761646.084173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761646.084176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761646.084199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761646.084207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761646/real 1789761646] req@ffff9811c92c4e00 x1876700048770176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761646.084215:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048770176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761646.084220:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761646.084222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761646.084229:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761646.339553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761646.979616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761647.363547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761648.003618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761648.387577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761649.027599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761649.411556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761650.051643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761650.435589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761651.075634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761651.203705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761651.203715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761651.203722:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761651.203735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1100 00000100:00080000:3.0:1789761651.203749:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761651.203759:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761651.203829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048770304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761651.203860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761651.203864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761651.203866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761651.203870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1100 00000100:00080000:3.0:1789761651.203875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761651.203879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761651.203890:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048770304:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761651.203890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761651.203913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048770432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761651.203929:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761651.203933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761651.203936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761651.203943:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1100 00000100:00080000:3.0:1789761651.203948:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761651.203962:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761651.203965:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789761651.203973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761651.203990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048770560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761651.204007:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761651/real 1789761651] req@ffff9811e1a20000 x1876700048770304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761651.204022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048770304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761651.204041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761651.204045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761651.204049:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048770304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761651.204071:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048770432:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761651.204088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761651.204094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761651.204096:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761651.204100:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761651.204108:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1100 00000100:00100000:1.0:1789761651.204109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761651/real 1789761651] req@ffff9811e1a23100 x1876700048770432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761651.204117:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761651.204122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048770432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761651.204124:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789761651.204135:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761651.204136:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761651.204140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761651.204143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048770432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761651.204151:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761651.204163:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761651.204170:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761651/real 1789761651] req@ffff9811e1a21f80 x1876700048770560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761651.204178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048770560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761651.204184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761651.204186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761651.204188:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761651.204287:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048770688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761651.204309:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761651.204313:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761651.204316:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761651.204322:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1100 00000100:00080000:3.0:1789761651.204328:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761651.204333:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761651.204356:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761651.204369:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761651/real 1789761651] req@ffff9811e1a20a80 x1876700048770688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761651.204378:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761651.204389:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761651.204397:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048770688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761651.204419:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761651.204422:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048770816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761651.204428:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761651.204437:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761651.204452:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761651.204499:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761651.204511:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761651/real 1789761651] req@ffff9811e1a20e00 x1876700048770816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761651.204521:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048770816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761651.204526:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761651.204529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761651.204542:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048770816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761651.459540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761652.099611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761652.483557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761653.123607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761653.507610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761654.147647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761654.531608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761655.171621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761655.555591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761656.195661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761656.323532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761656.323560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761656.323565:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761656.323574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1105 00000100:00080000:3.0:1789761656.323585:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761656.323592:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761656.323629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048770944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761656.323663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761656.323667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761656.323670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761656.323674:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1105 00000100:00080000:3.0:1789761656.323679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761656.323684:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761656.323697:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761656.323710:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048770944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761656.323717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048771072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761656.323745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761656.323749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761656.323751:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761656.323757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1105 00000100:00080000:3.0:1789761656.323762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761656.323767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761656.323774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761656.323789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048771200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761656.323803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761656.323808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761656.323809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761656.323813:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761656.323818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1105 00000100:00080000:3.0:1789761656.323822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761656.323826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761656.323831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761656.323841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048771328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761656.323848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761656.323851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789761656.323855:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761656/real 1789761656] req@ffff9811e1a21500 x1876700048770944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761656.323856:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761656.323862:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1105 00000100:00080000:3.0:1789761656.323868:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761656.323873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761656.323877:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761656.323882:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048770944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761656.323887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048771456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761656.323897:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761656.323903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761656.323906:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048770944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761656.323929:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048771072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761656.323946:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761656.323955:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761656/real 1789761656] req@ffff9811e1a20380 x1876700048771072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761656.323963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048771072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761656.323969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761656.323971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761656.323974:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048771072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761656.323978:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761656.323988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761656.323994:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761656/real 1789761656] req@ffff9811e1a20e00 x1876700048771200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761656.324010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048771200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761656.324016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761656.324018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761656.324020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761656.324024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761656.324037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761656.324043:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761656/real 1789761656] req@ffff9811e1a20a80 x1876700048771328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761656.324051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048771328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761656.324056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761656.324057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761656.324059:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761656.324062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761656.324072:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761656.324078:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761656/real 1789761656] req@ffff9811e1a21f80 x1876700048771456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761656.324084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048771456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761656.324098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761656.324100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761656.324102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761656.579601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761657.219596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761657.603597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761658.243614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761658.627609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761659.267602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761659.651584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761660.291618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761660.675546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761661.315607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761661.443562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761661.443571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761661.443575:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761661.443583:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1111 00000100:00080000:3.0:1789761661.443614:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761661.443623:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761661.443656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048771584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761661.443687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761661.443701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761661.443703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761661.443707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1111 00000100:00080000:3.0:1789761661.443711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761661.443714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761661.443722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761661.443732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048771712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761661.443743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761661.443747:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048771584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761661.443747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761661.443748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761661.443757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1111 00000100:00080000:3.0:1789761661.443760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761661.443762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761661.443765:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761661.443774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048771840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761661.443781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761661.443791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761661.443793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761661.443798:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1111 00000100:00080000:3.0:1789761661.443803:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761661.443808:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761661.443814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761661.443828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048771968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761661.443840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761661.443844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761661.443847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761661.443852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1111 00000100:00100000:1.0:1789761661.443856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048771712:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761661.443858:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761661.443865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761661.443870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761661.443883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048772096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761661.443901:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761661.443919:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761661.443938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761661.699542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761662.339622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761662.723574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761663.363632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761663.747559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761664.387615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761664.515752:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789761664.515804:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761664.515826:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761664.515875:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761664.515886:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761664.515897:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761664.515909:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789761664.515945:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761661/real 1789761664] req@ffff9811e1a20700 x1876700048772096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761664.515984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048772096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761664.515999:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761664.516006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761664.516033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761664.516042:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761661/real 1789761664] req@ffff9811e1a20000 x1876700048771712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761664.516058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048771712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761664.516068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761664.516072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761664.516076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048771712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761664.516083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761661/real 1789761664] req@ffff9811e1a23480 x1876700048771840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761664.516095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048771840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761664.516102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761664.516105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761664.516108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761664.516115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761661/real 1789761664] req@ffff9811e1a23800 x1876700048771968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761664.516126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048771968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761664.516133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761664.516136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761664.516148:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048771968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761664.516156:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761661/real 1789761664] req@ffff9811e1a23100 x1876700048771584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761664.516169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048771584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761664.516178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761664.516182:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761664.516185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048771584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761664.771540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761665.411603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761665.795587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761666.435636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761666.563561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761666.563570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761666.563573:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761666.563583:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1116 00000100:00080000:0.0:1789761666.563592:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761666.563598:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761666.563634:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048772224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761666.563675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761666.563678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761666.563680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761666.563683:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1116 00000100:00080000:0.0:1789761666.563686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761666.563690:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761666.563699:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761666.563715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048772352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761666.563741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048772224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761666.563745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761666.563751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761666.563753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761666.563758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1116 00000100:00080000:0.0:1789761666.563763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761666.563768:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761666.563775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761666.563790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048772480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761666.563799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761666.563802:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761666.563803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761666.563805:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1116 00000100:00080000:0.0:1789761666.563809:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761666.563815:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761666.563815:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761666.563820:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761666.563829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048772608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761666.563838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761666.563841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761666.563843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761666.563847:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1116 00000100:00080000:0.0:1789761666.563859:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761666.563866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761666.563873:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761666.563874:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761666/real 1789761666] req@ffff9811c2e29f80 x1876700048772224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761666.563898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048772224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761666.563906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048772736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761666.563915:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761666.563924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761666.563930:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048772224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761666.563960:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048772352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761666.563984:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761666.563997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761666/real 1789761666] req@ffff9811c2e28a80 x1876700048772352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761666.564019:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048772352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761666.564028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761666.564031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761666.564034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048772352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761666.564040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761666.564062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761666.564074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761666/real 1789761666] req@ffff9811c2e28380 x1876700048772480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761666.564102:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048772480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761666.564111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761666.564115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761666.564122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761666.564129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761666.564151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761666.564163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761666/real 1789761666] req@ffff9811c2e28e00 x1876700048772608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761666.564176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048772608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761666.564199:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761666.564203:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761666.564214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761666.564221:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761666.564245:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761666.564257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761666/real 1789761666] req@ffff9811c2e2bb80 x1876700048772736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761666.564270:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048772736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761666.564279:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761666.564282:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761666.564285:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048772736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761666.819589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761667.459626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761667.843535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761668.483661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761668.867581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761669.507649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761669.891589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761670.531653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761670.915527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761671.555678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761671.683568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761671.683580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761671.683584:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761671.683594:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1121 00000100:00080000:2.0:1789761671.683605:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761671.683611:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761671.683655:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048772864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761671.683676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761671.683679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761671.683680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761671.683683:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1121 00000100:00080000:2.0:1789761671.683685:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761671.683688:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761671.683695:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761671.683718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048772992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761671.683727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761671.683730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761671.683731:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761671.683734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1121 00000100:00080000:2.0:1789761671.683738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761671.683741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048772864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761671.683742:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761671.683749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761671.683758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048773120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761671.683765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761671.683768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761671.683769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761671.683771:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1121 00000100:00080000:2.0:1789761671.683773:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761671.683775:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761671.683779:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761671.683786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048773248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761671.683792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761671.683794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761671.683795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761671.683802:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1121 00000100:00080000:2.0:1789761671.683805:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761671.683811:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761671.683814:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761671.683816:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761671.683826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048773376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761671.683862:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761671/real 1789761671] req@ffff9811d863a300 x1876700048772864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761671.683883:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048772864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761671.683903:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761671.683908:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761671.683912:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048772864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761671.683937:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048772992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761671.683963:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761671.683978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761671/real 1789761671] req@ffff9811d8638700 x1876700048772992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761671.683992:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048772992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761671.684001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761671.684005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761671.684008:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048772992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761671.684015:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761671.684037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761671.684050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761671/real 1789761671] req@ffff9811d8638e00 x1876700048773120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761671.684063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048773120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761671.684073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761671.684076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761671.684080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761671.684094:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761671.684124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761671.684136:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761671/real 1789761671] req@ffff9811d8638380 x1876700048773248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761671.684149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048773248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761671.684158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761671.684162:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761671.684165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761671.684172:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761671.684193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761671.684205:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761671/real 1789761671] req@ffff9811d863b480 x1876700048773376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761671.684218:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048773376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761671.684227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761671.684230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761671.684234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761671.939537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761672.579643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761672.963579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761673.603648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761673.987581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761674.627604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761675.011620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761675.651640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761676.035607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761676.675630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761676.803569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761676.803593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761676.803597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761676.803610:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1126 00000100:00080000:2.0:1789761676.803624:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761676.803632:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761676.803676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048773504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761676.803702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761676.803705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761676.803706:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761676.803709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1126 00000100:00080000:2.0:1789761676.803712:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761676.803714:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761676.803723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761676.803732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048773632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761676.803741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761676.803758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048773504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761676.803763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761676.803767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761676.803771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1126 00000100:00080000:2.0:1789761676.803774:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761676.803777:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761676.803781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761676.803792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048773760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761676.803800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761676.803803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761676.803804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761676.803806:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1126 00000100:00080000:2.0:1789761676.803808:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761676.803810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761676.803814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761676.803823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048773888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761676.803829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761676.803832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761676.803834:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761676.803836:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1126 00000100:00080000:2.0:1789761676.803838:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761676.803840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789761676.803847:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761676.803851:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761676.803861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048774016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761676.803895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761676/real 1789761676] req@ffff9811d863b800 x1876700048773504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761676.803917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048773504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761676.803927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761676.803930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761676.803933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048773504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761676.803939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048773632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761676.803955:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761676.803963:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761676/real 1789761676] req@ffff9811d8638000 x1876700048773632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761676.803971:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048773632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761676.803977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761676.803979:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761676.803980:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048773632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761676.804002:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761676.804013:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761676.804020:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761676/real 1789761676] req@ffff9811d863aa00 x1876700048773760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761676.804036:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048773760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761676.804042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761676.804044:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761676.804046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761676.804049:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761676.804062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761676.804068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761676/real 1789761676] req@ffff9811d8638a80 x1876700048773888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761676.804076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048773888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761676.804081:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761676.804082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761676.804084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048773888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761676.804087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761676.804097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761676.804102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761676/real 1789761676] req@ffff9811d863a680 x1876700048774016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761676.804109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048774016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761676.804123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761676.804126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761676.804129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761677.059665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761677.699605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761678.083589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761678.723586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761679.107609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761679.747639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761680.131572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761680.771616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761681.155670:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761681.795622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761681.923594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761681.923609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761681.923614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761681.923628:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1131 00000100:00080000:3.0:1789761681.923642:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761681.923652:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761681.923700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048774144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761681.923746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761681.923749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761681.923751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761681.923754:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1131 00000100:00080000:3.0:1789761681.923757:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761681.923760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761681.923776:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761681.923792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048774272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761681.923800:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048774144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761681.923813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761681.923817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761681.923820:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761681.923825:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1131 00000100:00080000:3.0:1789761681.923830:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761681.923836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761681.923854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761681.923869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048774400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761681.923875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761681.923887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761681.923892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761681.923894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761681.923900:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1131 00000100:00080000:3.0:1789761681.923913:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761681.923918:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761681.923926:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761681.923927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761681/real 1789761681] req@ffff9811e1a22300 x1876700048774144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761681.923943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048774528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761681.923947:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048774144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761681.923975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761681.923976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761681.923983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761681.923984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761681.923988:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761681.923990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048774144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761681.923999:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1131 00000100:00080000:3.0:1789761681.924005:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761681.924010:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761681.924017:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761681.924018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048774272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761681.924037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048774656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761681.924045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761681.924059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761681/real 1789761681] req@ffff9811e1a21180 x1876700048774272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761681.924069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048774272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761681.924083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761681.924086:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761681.924089:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048774272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761681.924095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761681.924119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761681.924138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761681/real 1789761681] req@ffff9811e1a22a00 x1876700048774400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761681.924153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048774400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761681.924161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761681.924163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761681.924166:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761681.924170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761681.924188:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761681.924196:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761681/real 1789761681] req@ffff9811e1a21c00 x1876700048774528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761681.924204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048774528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761681.924212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761681.924215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761681.924227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761681.924243:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761681.924266:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761681.924275:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761681/real 1789761681] req@ffff9811e1a21880 x1876700048774656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761681.924283:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048774656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761681.924289:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761681.924291:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761681.924293:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048774656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761682.179605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761682.819665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761683.203666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761683.843653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761684.227599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761684.867650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761685.251600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761685.891635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761686.275577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761686.915664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761687.043558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761687.043570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761687.043575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761687.043587:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1136 00000100:00080000:2.0:1789761687.043599:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761687.043608:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761687.043666:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048774784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761687.043703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761687.043706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761687.043708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761687.043713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1136 00000100:00080000:2.0:1789761687.043717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761687.043721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761687.043731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761687.043745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048774912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761687.043753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048774784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761687.043757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761687.043763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761687.043765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761687.043768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1136 00000100:00080000:2.0:1789761687.043784:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761687.043788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761687.043793:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761687.043811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048775040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761687.043819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761687.043821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761687.043823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761687.043825:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1136 00000100:00080000:2.0:1789761687.043874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761687.043880:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761687.043884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761687.043895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048775168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761687.043895:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761687.043905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761687.043908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761687.043909:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761687.043911:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1136 00000100:00080000:2.0:1789761687.043914:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761687.043916:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761687.043919:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761687.043925:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048775296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761687.044096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761687/real 1789761687] req@ffff9811d8639f80 x1876700048774784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761687.044115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048774784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761687.044126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761687.044130:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761687.044136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048774784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761687.044173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048774912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761687.044199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761687.044214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761687/real 1789761687] req@ffff9811d8639500 x1876700048774912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761687.044234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048774912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761687.044243:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761687.044246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761687.044250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048774912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761687.044256:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761687.044287:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761687.044299:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761687/real 1789761687] req@ffff9811d8639880 x1876700048775040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761687.044312:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048775040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761687.044320:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761687.044323:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761687.044327:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761687.044334:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761687.044356:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761687.044368:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761687/real 1789761687] req@ffff9811d863ad80 x1876700048775168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761687.044388:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048775168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761687.044396:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761687.044400:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761687.044403:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761687.044409:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761687.044432:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761687.044442:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761687/real 1789761687] req@ffff9811d8639c00 x1876700048775296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761687.044450:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048775296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761687.044456:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761687.044458:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761687.044489:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761687.299598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761687.939650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761688.323614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761688.963681:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761689.347562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761689.987673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761690.371588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761691.011654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761691.395586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761692.035613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761692.163611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761692.163625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761692.163631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761692.163644:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1141 00000100:00080000:0.0:1789761692.163657:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761692.163665:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761692.163717:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048775424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761692.163750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761692.163753:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761692.163755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761692.163758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1141 00000100:00080000:0.0:1789761692.163761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761692.163765:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761692.163775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761692.163787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048775552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761692.163798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761692.163798:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048775424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761692.163808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761692.163809:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761692.163813:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1141 00000100:00080000:0.0:1789761692.163815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761692.163817:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761692.163820:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761692.163854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048775680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761692.163865:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761692.163869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761692.163871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761692.163875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761692.163880:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1141 00000100:00080000:0.0:1789761692.163887:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761692.163894:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761692.163900:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761692.163912:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048775808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761692.163920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761692.163923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761692.163927:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761692.163928:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761692/real 1789761692] req@ffff9811c2e29180 x1876700048775424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761692.163935:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1141 00000100:00080000:0.0:1789761692.163941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761692.163947:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761692.163951:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761692.163955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048775424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761692.163961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048775936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761692.163987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761692.163993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761692.163999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048775424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761692.164028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048775552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761692.164057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761692.164082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761692/real 1789761692] req@ffff9811c2e2aa00 x1876700048775552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761692.164096:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048775552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761692.164105:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761692.164109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761692.164112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048775552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761692.164119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789761692.164136:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761692.164146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761692/real 1789761692] req@ffff9811c2e2b100 x1876700048775680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761692.164158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048775680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761692.164167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761692.164170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761692.164173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761692.164185:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761692.164202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761692.164213:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761692/real 1789761692] req@ffff9811c2e29880 x1876700048775808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761692.164226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048775808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761692.164234:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761692.164238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761692.164241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761692.164247:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789761692.164270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761692.164281:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761692/real 1789761692] req@ffff9811c2e2ad80 x1876700048775936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761692.164293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048775936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761692.164302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761692.164306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789761692.164309:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048775936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761692.419635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761693.059618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761693.443561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761694.083598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761694.467649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761695.107605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761695.491553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761696.131629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761696.515632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761697.155612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761697.283560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761697.283586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761697.283590:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761697.283600:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1146 00000100:00080000:3.0:1789761697.283609:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761697.283615:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761697.283643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048776064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761697.283668:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761697.283674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761697.283675:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761697.283678:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1146 00000100:00080000:3.0:1789761697.283681:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761697.283683:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761697.283692:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761697.283700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048776192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761697.283709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761697.283711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761697.283713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761697.283716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1146 00000100:00080000:3.0:1789761697.283717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761697.283721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761697.283724:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761697.283743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048776064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761697.283743:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048776320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761697.283767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761697.283769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761697.283771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761697.283775:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1146 00000100:00080000:3.0:1789761697.283778:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761697.283781:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761697.283786:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761697.283794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048776448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761697.283801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761697.283804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761697.283805:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761697.283808:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1146 00000100:00080000:3.0:1789761697.283810:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761697.283814:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761697.283817:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761697.283824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048776576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761697.283849:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761697.283896:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761697/real 1789761697] req@ffff9811e1a22d80 x1876700048776064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761697.283925:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048776064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761697.283935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761697.283939:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761697.283944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048776064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761697.283953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048776192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761697.283980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761697.283995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761697/real 1789761697] req@ffff9811e1a22680 x1876700048776192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761697.284009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048776192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761697.284017:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761697.284021:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761697.284025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048776192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761697.284055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761697.284075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761697.284084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761697/real 1789761697] req@ffff9811e1a21880 x1876700048776320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761697.284092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048776320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761697.284108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761697.284111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761697.284113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761697.284117:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761697.284133:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761697.284140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761697/real 1789761697] req@ffff9811e1a21c00 x1876700048776448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761697.284147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048776448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761697.284152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761697.284164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761697.284168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761697.284175:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761697.284197:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761697.284210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761697/real 1789761697] req@ffff9811e1a22a00 x1876700048776576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761697.284223:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048776576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761697.284232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761697.284236:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761697.284250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761697.539550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761698.179649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761698.563586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761699.203623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761699.587663:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761700.227599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761700.611582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761701.251644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761701.635546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761702.275615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761702.403591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761702.403600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761702.403603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761702.403611:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1151 00000100:00080000:0.0:1789761702.403620:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761702.403626:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761702.403657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048776704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761702.403681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761702.403683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761702.403685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761702.403688:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1151 00000100:00080000:0.0:1789761702.403692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761702.403696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761702.403706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761702.403721:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048776832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761702.403732:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048776704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761702.403736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761702.403742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761702.403757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761702.403763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1151 00000100:00080000:0.0:1789761702.403778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761702.403782:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761702.403787:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761702.403787:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761702.403801:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048776960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761702.403809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761702.403812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761702.403813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761702.403816:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1151 00000100:00080000:0.0:1789761702.403819:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761702.403822:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761702.403826:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761702/real 1789761702] req@ffff9811c2e28700 x1876700048776704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761702.403828:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761702.403843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048777088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761702.403848:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048776704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761702.403856:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761702.403863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761702.403865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761702.403866:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761702.403870:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1151 00000100:00080000:1.0:1789761702.403874:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789761702.403875:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761702.403882:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761702.403882:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048776704:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761702.403895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761702.403903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048776832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761702.403911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048777216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761702.403924:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761702.403947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761702/real 1789761702] req@ffff9811c2e2a680 x1876700048776832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761702.403957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048776832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761702.403963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761702.403966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761702.403968:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048776832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761702.403972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761702.403995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761702.404003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761702/real 1789761702] req@ffff9811c2e29c00 x1876700048776960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761702.404011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048776960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761702.404016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761702.404018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761702.404020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048776960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761702.404024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761702.404034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761702.404040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761702/real 1789761702] req@ffff9811c2e2ad80 x1876700048777088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761702.404047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048777088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761702.404052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761702.404054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761702.404056:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761702.404067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761702.404077:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761702.404083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761702/real 1789761702] req@ffff9811c2e29880 x1876700048777216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761702.404101:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048777216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761702.404107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761702.404109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761702.404111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761702.659558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761703.299641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761703.683533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761704.323611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761704.707622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761705.347622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761705.731549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761706.371632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761706.755644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761707.395656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761707.523619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761707.523632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761707.523637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761707.523651:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1157 00000100:00080000:0.0:1789761707.523665:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761707.523674:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761707.523729:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048777344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761707.523761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761707.523765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761707.523767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761707.523772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1157 00000100:00080000:0.0:1789761707.523775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761707.523779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761707.523790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761707.523803:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048777344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761707.523804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048777472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761707.523819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761707.523841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761707.523842:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761707.523846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1157 00000100:00080000:0.0:1789761707.523864:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761707.523868:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761707.523872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761707.523877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761707.523885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048777600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761707.523894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761707.523898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761707.523899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761707.523902:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1157 00000100:00080000:0.0:1789761707.523905:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761707.523907:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761707.523911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761707.523924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048777728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761707.523924:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761707/real 1789761707] req@ffff9811c2e2b100 x1876700048777344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761707.523953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761707.523957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761707.523959:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761707.523962:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1157 00000100:00100000:1.0:1789761707.523966:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048777344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761707.523967:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761707.523974:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789761707.523981:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761707.523984:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761707.523987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761707.523993:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048777344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761707.523999:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048777856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761707.524026:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048777472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761707.524055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761707.524070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761707/real 1789761707] req@ffff9811c2e2aa00 x1876700048777472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761707.524095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048777472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761707.524102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761707.524105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761707.524107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048777472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761707.524112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761707.524129:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761707.524140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761707/real 1789761707] req@ffff9811c2e29180 x1876700048777600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761707.524153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048777600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761707.524161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761707.524165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761707.524169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761707.524176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761707.524196:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761707.524204:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761707/real 1789761707] req@ffff9811c2e2bb80 x1876700048777728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761707.524212:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048777728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761707.524241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761707.524243:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761707.524245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761707.524250:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761707.524271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761707.524279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761707/real 1789761707] req@ffff9811c2e28e00 x1876700048777856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761707.524287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048777856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761707.524292:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761707.524294:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761707.524296:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048777856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761707.779537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761708.419614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761708.803644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761709.443661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761709.827571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761710.467648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761710.851607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761711.491627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761711.875539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761712.515655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761712.643603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761712.643618:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761712.643624:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761712.643639:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1162 00000100:00080000:0.0:1789761712.643653:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761712.643663:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761712.643711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048777984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761712.643735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761712.643738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761712.643740:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761712.643743:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1162 00000100:00080000:0.0:1789761712.643747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761712.643750:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761712.643760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761712.643771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048778112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761712.643776:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048777984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761712.643807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761712.643810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761712.643812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761712.643816:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1162 00000100:00080000:0.0:1789761712.643842:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761712.643845:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761712.643849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789761712.643857:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761712.643860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048778240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761712.643869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761712.643872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761712.643873:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761712.643876:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1162 00000100:00080000:0.0:1789761712.643881:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761712.643885:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761712.643890:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761712.643903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048778368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761712.643914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761712.643916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761712/real 1789761712] req@ffff9811c2e28380 x1876700048777984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761712.643919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761712.643923:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761712.643934:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1162 00000100:00100000:1.0:1789761712.643943:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048777984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761712.643948:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761712.643954:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761712.643960:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789761712.643972:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761712.643977:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761712.643982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048778496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761712.643983:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048777984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761712.644018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048778112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761712.644039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761712.644050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761712/real 1789761712] req@ffff9811c2e28a80 x1876700048778112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761712.644058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048778112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761712.644064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761712.644067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761712.644069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048778112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761712.644073:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761712.644084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761712.644090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761712/real 1789761712] req@ffff9811c2e29f80 x1876700048778240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761712.644099:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048778240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761712.644107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761712.644111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761712.644114:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761712.644127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761712.644158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761712.644167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761712/real 1789761712] req@ffff9811c2e2b800 x1876700048778368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761712.644175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048778368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761712.644181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761712.644183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761712.644185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761712.644189:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761712.644201:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761712.644206:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761712/real 1789761712] req@ffff9811c2e2b480 x1876700048778496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761712.644213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048778496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761712.644221:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761712.644225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761712.644228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761712.899594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761713.539594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761713.923590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761714.563637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761714.947674:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761715.587593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761715.971616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761716.611686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761716.995623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761717.635643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761717.763560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761717.763593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761717.763599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761717.763612:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1167 00000100:00080000:3.0:1789761717.763626:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761717.763637:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761717.763682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048778624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761717.763715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761717.763720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761717.763722:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761717.763726:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1167 00000100:00080000:3.0:1789761717.763731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761717.763737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761717.763748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761717.763758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048778624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761717.763762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048778752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761717.763773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761717.763776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761717.763777:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761717.763781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1167 00000100:00080000:3.0:1789761717.763783:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761717.763786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761717.763789:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761717.763799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048778880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761717.763811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761717.763814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761717.763815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761717.763818:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1167 00000100:00080000:3.0:1789761717.763822:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761717.763825:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761717.763828:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761717.763839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048779008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789761717.763840:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761717.763848:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761717.763851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761717.763853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761717.763855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1167 00000100:00080000:3.0:1789761717.763857:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761717.763860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761717.763863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761717.763868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048779136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761717.763905:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761717/real 1789761717] req@ffff9811e1a21180 x1876700048778624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761717.763922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048778624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761717.763931:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761717.763935:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761717.763938:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048778624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761717.763961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048778752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761717.763978:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761717.763987:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761717/real 1789761717] req@ffff9811e1a22300 x1876700048778752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761717.764001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048778752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761717.764010:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761717.764014:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761717.764018:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048778752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761717.764024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761717.764045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761717.764054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761717/real 1789761717] req@ffff9811e1a23100 x1876700048778880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761717.764070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048778880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761717.764076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761717.764078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761717.764080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048778880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761717.764086:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761717.764108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761717.764120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761717/real 1789761717] req@ffff9811e1a23800 x1876700048779008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761717.764133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048779008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761717.764141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761717.764144:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761717.764146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761717.764150:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761717.764166:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761717.764174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761717/real 1789761717] req@ffff9811e1a23480 x1876700048779136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761717.764189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048779136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761717.764204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761717.764207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761717.764209:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761718.019607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761718.659616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761719.043581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761719.683637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761720.067676:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761720.707615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761721.091547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761721.731665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761722.115610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761722.755571:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761722.883574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761722.883584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761722.883603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761722.883616:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1172 00000100:00080000:3.0:1789761722.883627:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761722.883635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761722.883674:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048779264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761722.883703:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761722.883706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761722.883708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761722.883712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1172 00000100:00080000:3.0:1789761722.883718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761722.883724:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761722.883734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761722.883741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048779264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761722.883749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048779392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761722.883766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761722.883772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761722.883776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761722.883781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1172 00000100:00080000:3.0:1789761722.883787:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761722.883792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761722.883802:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761722.883806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761722.883822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048779520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761722.883846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761722.883847:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761722/real 1789761722] req@ffff9811e1a20000 x1876700048779264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761722.883856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789761722.883870:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048779264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761722.883879:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761722.883882:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761722.883884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048779264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761722.883903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048779392:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761722.883921:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761722.883929:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1172 00000800:00000100:0.0:1789761722.883930:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761722.883934:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761722.883939:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761722.883945:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761722.883951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761722/real 1789761722] req@ffff9811e1a20700 x1876700048779392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761722.883958:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048779648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761722.883978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048779392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761722.883988:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761722.883993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761722.883997:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048779392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761722.884004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761722.884024:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761722.884033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761722/real 1789761722] req@ffff9811e1a21f80 x1876700048779520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761722.884041:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048779520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761722.884047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761722.884050:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761722.884052:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779520:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761722.884108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761722.884113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761722.884116:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761722.884174:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1172 00000100:00080000:3.0:1789761722.884179:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761722.884183:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761722.884201:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761722.884216:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048779776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761722.884263:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761722.884303:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761722.884312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761722/real 1789761722] req@ffff9811e1a20a80 x1876700048779648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761722.884331:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048779648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761722.884337:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761722.884339:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761722.884341:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761722.884345:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761722.884358:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761722.884364:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761722/real 1789761722] req@ffff9811e1a20e00 x1876700048779776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761722.884371:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048779776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761722.884376:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761722.884378:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761722.884380:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048779776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761723.139554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761723.779608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761724.163589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761724.803589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761725.187595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761725.827658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761726.211620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761726.851598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761727.235537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761727.875667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761728.003565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761728.003576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761728.003580:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761728.003590:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1177 00000100:00080000:1.0:1789761728.003601:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761728.003609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761728.003657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048779904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761728.003680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761728.003684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761728.003685:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761728.003688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1177 00000100:00080000:1.0:1789761728.003691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761728.003693:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761728.003702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761728.003710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048780032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761728.003737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761728.003741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761728.003742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761728.003746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1177 00000100:00080000:1.0:1789761728.003749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761728.003751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761728.003756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761728.003765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048780160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761728.003778:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048779904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761728.003781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761728.003798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761728.003800:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761728.003804:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1177 00000100:00080000:1.0:1789761728.003806:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761728.003809:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761728.003812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761728.003823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048780288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761728.003830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761728.003833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761728.003835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761728.003838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1177 00000100:00080000:1.0:1789761728.003853:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761728.003857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761728.003862:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761728.003874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048780416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761728.003906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048780032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761728.003959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761728.003987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761728.004016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761728.259599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761728.899647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761729.283563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761729.923622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789761730.307565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761730.947624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761731.075720:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789761731.075778:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761731.075794:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761731.075828:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761731.075834:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761731.075839:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789761731.075845:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789761731.075925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761728/real 1789761731] req@ffff9811c92c4000 x1876700048780160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761731.075965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048780160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761731.075979:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761731.075984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761731.075989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761731.075999:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761728/real 1789761731] req@ffff9811c92c6680 x1876700048780288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761731.076012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048780288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761731.076021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761731.076025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761731.076030:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761731.076038:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761728/real 1789761731] req@ffff9811c92c5c00 x1876700048780416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761731.076074:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048780416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761731.076082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761731.076086:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761731.076090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761731.076098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761728/real 1789761731] req@ffff9811c92c5880 x1876700048779904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761731.076110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048779904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761731.076119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761731.076122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761731.076125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048779904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761731.076133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761728/real 1789761731] req@ffff9811c92c5500 x1876700048780032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761731.076145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048780032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761731.076153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761731.076156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761731.076159:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048780032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761731.331583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761731.971692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761732.355576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761732.995600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761733.123563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761733.123572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761733.123576:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761733.123586:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1182 00000100:00080000:1.0:1789761733.123596:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761733.123602:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761733.123631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048780544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761733.123662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761733.123667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761733.123669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761733.123673:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1182 00000100:00080000:1.0:1789761733.123678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761733.123683:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761733.123693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761733.123706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048780672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761733.123711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048780544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761733.123742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761733.123752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761733.123755:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761733.123761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1182 00000100:00080000:1.0:1789761733.123766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761733.123771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761733.123778:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761733.123796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048780800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761733.123797:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761733.123811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761733.123815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761733.123818:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761733.123823:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1182 00000100:00080000:1.0:1789761733.123827:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761733.123832:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761733.123838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761733.123845:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761733/real 1789761733] req@ffff9811c92c6d80 x1876700048780544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761733.123855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048780928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761733.123872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761733.123877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761733.123879:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761733.123883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1182 00000100:00100000:0.0:1789761733.123889:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048780544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761733.123898:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761733.123906:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761733.123913:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761733.123918:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761733.123926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761733.123936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048781056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761733.123941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048780544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761733.123983:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048780672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761733.124010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761733.124031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761733/real 1789761733] req@ffff9811c92c5180 x1876700048780672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761733.124045:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048780672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761733.124054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761733.124058:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761733.124062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048780672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761733.124068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761733.124093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761733.124105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761733/real 1789761733] req@ffff9811c92c7b80 x1876700048780800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761733.124135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048780800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761733.124143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761733.124146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761733.124150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761733.124171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761733.124197:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761733.124208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761733/real 1789761733] req@ffff9811c92c5500 x1876700048780928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761733.124220:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048780928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761733.124229:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761733.124232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761733.124235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048780928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761733.124242:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761733.124261:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761733.124272:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761733/real 1789761733] req@ffff9811c92c5880 x1876700048781056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761733.124285:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048781056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761733.124307:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761733.124311:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761733.124315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761733.379534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761734.019594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761734.403533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761735.043625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761735.427515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761736.067630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761736.451534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761737.091655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761737.475518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761738.115656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761738.243573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761738.243584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761738.243587:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761738.243595:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1187 00000100:00080000:3.0:1789761738.243605:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761738.243611:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761738.243643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048781184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761738.243664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761738.243666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761738.243667:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761738.243670:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1187 00000100:00080000:3.0:1789761738.243672:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761738.243674:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761738.243682:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761738.243690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048781312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761738.243699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761738.243707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761738.243709:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048781184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761738.243709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761738.243714:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1187 00000100:00080000:3.0:1789761738.243717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761738.243719:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761738.243736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761738.243746:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048781440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761738.243757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761738.243760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761738.243762:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761738.243767:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1187 00000100:00080000:3.0:1789761738.243773:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761738.243778:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761738.243783:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761738.243786:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761738.243801:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048781568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761738.243811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761738.243814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761738.243816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761738.243821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1187 00000100:00080000:3.0:1789761738.243825:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761738.243827:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761738/real 1789761738] req@ffff9811e1a20380 x1876700048781184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761738.243834:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761738.243844:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761738.243850:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048781184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761738.243860:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761738.243860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048781696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761738.243869:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761738.243876:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048781184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761738.243899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048781312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761738.243914:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761738.243922:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761738/real 1789761738] req@ffff9811e1a21500 x1876700048781312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761738.243931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048781312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761738.243937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761738.243945:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761738.243948:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048781312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761738.243953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761738.243967:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761738.243974:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761738/real 1789761738] req@ffff9811e1a20e00 x1876700048781440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761738.243982:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048781440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761738.243987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761738.243989:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761738.243991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761738.243994:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761738.244004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761738.244009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761738/real 1789761738] req@ffff9811e1a20a80 x1876700048781568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761738.244017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048781568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761738.244022:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761738.244024:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761738.244027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761738.244032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761738.244059:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761738.244071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761738/real 1789761738] req@ffff9811e1a21f80 x1876700048781696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761738.244085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048781696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761738.244091:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761738.244093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761738.244095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048781696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761738.499525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761739.139613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761739.523519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761740.163649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761740.547559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761741.187644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761741.571590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761742.211668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761742.595541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761743.235615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761743.363633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761743.363647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761743.363652:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761743.363666:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1192 00000100:00080000:0.0:1789761743.363681:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761743.363691:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761743.363754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048781824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761743.363793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761743.363798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761743.363800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761743.363804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1192 00000100:00080000:0.0:1789761743.363808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761743.363813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761743.363844:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048781824:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761743.363845:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761743.363867:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048781952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761743.363882:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761743.363886:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761743.363888:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761743.363893:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1192 00000100:00080000:0.0:1789761743.363898:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761743.363903:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761743.363908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761743.363921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048782080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761743.363931:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761743.363939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761743.363944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761743.363946:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761743.363952:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1192 00000100:00080000:0.0:1789761743.363957:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761743.363961:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761743.363968:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761743.363983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048782208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761743.363996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761743.364000:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761743/real 1789761743] req@ffff9811c2e2a300 x1876700048781824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761743.364001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761743.364012:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761743.364018:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1192 00000100:00100000:2.0:1789761743.364022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048781824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761743.364026:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761743.364035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761743.364037:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761743.364043:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761743.364047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048781824:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761743.364048:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761743.364068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048782336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761743.364074:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048781952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761743.364117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761743.364132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761743/real 1789761743] req@ffff9811c2e28000 x1876700048781952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761743.364155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048781952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761743.364179:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761743.364182:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761743.364185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048781952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761743.364190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761743.364208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761743.364216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761743/real 1789761743] req@ffff9811c2e29500 x1876700048782080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761743.364224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048782080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761743.364230:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761743.364233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761743.364235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761743.364238:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761743.364249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761743.364255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761743/real 1789761743] req@ffff9811c2e2b480 x1876700048782208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761743.364262:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048782208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761743.364267:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761743.364269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761743.364277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761743.364289:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761743.364303:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761743.364310:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761743/real 1789761743] req@ffff9811c2e2b800 x1876700048782336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761743.364318:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048782336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761743.364323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761743.364326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761743.364327:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761743.619597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761744.259671:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761744.643566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761745.283615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761745.667578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761746.307603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761746.691539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761747.331596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761747.715615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761748.355607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761748.483629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761748.483643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761748.483648:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761748.483659:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1198 00000100:00080000:3.0:1789761748.483676:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761748.483684:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761748.483725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048782464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761748.483750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761748.483752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761748.483754:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761748.483757:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1198 00000100:00080000:3.0:1789761748.483764:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761748.483768:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761748.483783:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761748.483801:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048782592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761748.483811:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048782464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761748.483819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761748.483825:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761748.483827:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761748.483832:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1198 00000100:00080000:3.0:1789761748.483837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761748.483842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761748.483869:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761748.483885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048782720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761748.483897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761748.483903:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761748.483904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761748.483909:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761748.483915:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1198 00000100:00080000:3.0:1789761748.483921:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761748.483928:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761748.483934:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761748.483949:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048782848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761748.483960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761748.483964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761748.483966:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761748.483971:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1198 00000100:00080000:3.0:1789761748.483976:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761748.483980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761748/real 1789761748] req@ffff9811e1a20700 x1876700048782464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761748.483985:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761748.483995:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761748.484010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048782976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761748.484013:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048782464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761748.484036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761748.484056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761748.484062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048782464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761748.484092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048782592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761748.484120:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761748.484145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761748/real 1789761748] req@ffff9811e1a20000 x1876700048782592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761748.484160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048782592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761748.484170:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761748.484172:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761748.484175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048782592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761748.484179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761748.484197:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761748.484210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761748/real 1789761748] req@ffff9811e1a23480 x1876700048782720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761748.484223:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048782720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761748.484231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761748.484235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761748.484238:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761748.484245:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761748.484270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761748.484283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761748/real 1789761748] req@ffff9811e1a23800 x1876700048782848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761748.484295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048782848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761748.484311:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761748.484315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761748.484319:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761748.484325:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761748.484349:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761748.484358:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761748/real 1789761748] req@ffff9811e1a23100 x1876700048782976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761748.484366:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048782976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761748.484372:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761748.484375:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761748.484378:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048782976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761748.739584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761749.379667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761749.763571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761750.403629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761750.787586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761751.427672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761751.811547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761752.451630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761752.835558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761753.475673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761753.603544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761753.603555:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761753.603560:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761753.603571:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1203 00000100:00080000:3.0:1789761753.603582:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761753.603591:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761753.603627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048783104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761753.603659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761753.603663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761753.603665:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761753.603670:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1203 00000100:00080000:3.0:1789761753.603673:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761753.603678:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761753.603688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761753.603702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048783232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761753.603706:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048783104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761753.603724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761753.603729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761753.603731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761753.603737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1203 00000100:00080000:3.0:1789761753.603742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761753.603746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761753.603751:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761753.603777:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761753.603796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048783360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761753.603808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761753.603812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789761753.603817:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761753/real 1789761753] req@ffff9811e1a22300 x1876700048783104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761753.603817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761753.603828:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1203 00000100:00080000:3.0:1789761753.603836:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761753.603841:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048783104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761753.603845:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761753.603857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761753.603858:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761753.603864:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761753.603868:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048783104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761753.603887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048783488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761753.603899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761753.603903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761753.603905:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761753.603910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048783232:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761753.603912:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1203 00000100:00080000:3.0:1789761753.603919:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761753.603924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761753.603930:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761753.603937:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761753.603947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048783616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761753.603953:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761753/real 1789761753] req@ffff9811e1a21180 x1876700048783232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761753.603981:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048783232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761753.603992:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761753.603997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761753.604016:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048783232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761753.604024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761753.604045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761753.604058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761753/real 1789761753] req@ffff9811e1a22a00 x1876700048783360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761753.604080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048783360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761753.604089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761753.604092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761753.604096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761753.604102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761753.604124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761753.604135:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761753/real 1789761753] req@ffff9811e1a21c00 x1876700048783488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761753.604156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048783488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761753.604166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761753.604168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761753.604170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761753.604183:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761753.604199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761753.604206:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761753/real 1789761753] req@ffff9811e1a21880 x1876700048783616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761753.604214:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048783616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761753.604220:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761753.604222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761753.604224:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048783616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761753.859551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761754.499661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761754.883572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761755.523680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761755.907556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761756.547557:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761756.931552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761757.571612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761757.955532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761758.595647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761758.723666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761758.723675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761758.723697:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761758.723711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1208 00000100:00080000:3.0:1789761758.723725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761758.723735:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761758.723777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048783744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761758.723802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761758.723807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761758.723809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761758.723813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1208 00000100:00080000:3.0:1789761758.723817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761758.723821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761758.723832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761758.723840:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048783744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761758.723847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048783872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761758.723868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761758.723872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761758.723874:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761758.723879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1208 00000100:00080000:3.0:1789761758.723883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761758.723888:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761758.723894:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761758.723939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761758.724013:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761758/real 1789761758] req@ffff9811e1a22680 x1876700048783744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761758.724018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048784000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761758.724041:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048783744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761758.724078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761758.724084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761758.724089:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048783744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761758.724121:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048783872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761758.724170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761758.724205:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761758/real 1789761758] req@ffff9811e1a22d80 x1876700048783872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761758.724219:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048783872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761758.724233:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761758.724238:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761758.724243:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761758.724245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761758.724262:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761758.724267:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1208 00000100:00100000:2.0:1789761758.724270:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048783872:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761758.724283:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761758.724305:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784000:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761758.724311:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761758.724320:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761758.724333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761758.724337:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048784128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761758.724344:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761758/real 1789761758] req@ffff9811e1a21880 x1876700048784000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761758.724351:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761758.724370:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761758.724373:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761758.724377:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048784000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761758.724383:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1208 00000100:00080000:2.0:1789761758.724392:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761758.724396:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761758.724397:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761758.724405:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784000:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761758.724411:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761758.724428:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761758.724436:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761758.724442:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048784256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761758.724483:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761758.724508:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761758/real 1789761758] req@ffff9811e1a21c00 x1876700048784128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761758.724530:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048784128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761758.724542:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761758.724549:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761758.724556:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761758.724579:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761758.724601:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761758.724621:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761758/real 1789761758] req@ffff9811e1a22a00 x1876700048784256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761758.724636:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048784256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761758.724642:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761758.724644:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761758.724646:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761758.979611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761759.619664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761760.003525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761760.643609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761761.027643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761761.667652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761762.051607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761762.691675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761763.075709:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761763.715640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761763.843548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761763.843557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761763.843560:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761763.843568:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1213 00000100:00080000:3.0:1789761763.843594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761763.843601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761763.843629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048784384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761763.843655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761763.843658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761763.843660:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761763.843662:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1213 00000100:00080000:3.0:1789761763.843664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761763.843666:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761763.843674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761763.843682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048784512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761763.843690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761763.843693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761763.843694:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761763.843697:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1213 00000100:00080000:3.0:1789761763.843699:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761763.843701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761763.843704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761763.843711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048784384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761763.843712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048784640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761763.843732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761763.843740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761763.843741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761763.843745:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1213 00000100:00080000:3.0:1789761763.843748:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761763.843751:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761763.843756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761763.843765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048784768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761763.843772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761763.843774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761763.843775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761763.843777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1213 00000100:00080000:3.0:1789761763.843779:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761763.843784:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761763.843785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761763.843790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761763.843796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048784896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761763.843836:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761763/real 1789761763] req@ffff9811e1a21180 x1876700048784384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761763.843860:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048784384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761763.843867:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761763.843871:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761763.843875:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048784384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761763.843881:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048784512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761763.843901:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761763.843909:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761763/real 1789761763] req@ffff9811e1a22300 x1876700048784512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761763.843917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048784512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761763.843923:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761763.843938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761763.843941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048784512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761763.843960:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761763.843983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761763.843990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761763/real 1789761763] req@ffff9811e1a23100 x1876700048784640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761763.843998:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048784640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761763.844020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761763.844022:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761763.844025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761763.844029:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761763.844043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761763.844050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761763/real 1789761763] req@ffff9811e1a23800 x1876700048784768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761763.844058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048784768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761763.844063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761763.844065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761763.844067:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761763.844070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761763.844080:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761763.844085:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761763/real 1789761763] req@ffff9811e1a23480 x1876700048784896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761763.844092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048784896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761763.844097:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761763.844099:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761763.844106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048784896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761764.099545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761764.739613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761765.123543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761765.763640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761766.147569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761766.787661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761767.171574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761767.811626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761768.195543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761768.835621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761768.963598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761768.963610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761768.963616:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761768.963629:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1218 00000100:00080000:0.0:1789761768.963643:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761768.963652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761768.963704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048785024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761768.963732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761768.963735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761768.963737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761768.963740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1218 00000100:00080000:0.0:1789761768.963743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761768.963746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761768.963756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761768.963771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048785152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761768.963791:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048785024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761768.963818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761768.963822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761768.963824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761768.963831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1218 00000100:00080000:0.0:1789761768.963837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761768.963843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761768.963851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761768.963862:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761768.963870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048785280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761768.963883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761768.963900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761768.963903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761768.963909:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1218 00000100:00080000:0.0:1789761768.963915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761768.963920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761768.963920:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761768/real 1789761768] req@ffff9811c2e29f80 x1876700048785024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761768.963932:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761768.963954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048785408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761768.963955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048785024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761768.964021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761768.964022:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761768.964031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761768.964037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789761768.964038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761768.964060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1218 00000100:00100000:2.0:1789761768.964061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048785024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761768.964072:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761768.964078:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761768.964083:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761768.964093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048785152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761768.964098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048785536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761768.964123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761768.964134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761768/real 1789761768] req@ffff9811c2e28a80 x1876700048785152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761768.964145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048785152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761768.964153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761768.964156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761768.964158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048785152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761768.964163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761768.964175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761768.964196:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761768/real 1789761768] req@ffff9811c2e28380 x1876700048785280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761768.964204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048785280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761768.964210:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761768.964212:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761768.964214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761768.964218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761768.964231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761768.964236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761768/real 1789761768] req@ffff9811c2e28e00 x1876700048785408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761768.964245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048785408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761768.964254:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761768.964257:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761768.964260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761768.964279:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761768.964300:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761768.964312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761768/real 1789761768] req@ffff9811c2e2bb80 x1876700048785536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761768.964324:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048785536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761768.964339:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761768.964342:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761768.964346:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761769.219585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761769.859696:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761770.243524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761770.883650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761771.267559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761771.907588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761772.291516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761772.931613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761773.315571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761773.955625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761774.083576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761774.083586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761774.083591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761774.083601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1223 00000100:00080000:3.0:1789761774.083616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761774.083625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761774.083674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048785664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761774.083710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761774.083714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761774.083716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761774.083720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1223 00000100:00080000:3.0:1789761774.083724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761774.083729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761774.083739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761774.083753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048785792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761774.083761:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048785664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761774.083769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761774.083774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761774.083776:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761774.083780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1223 00000100:00080000:3.0:1789761774.083784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761774.083789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761774.083793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761774.083805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048785920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761774.083816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761774.083820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761774.083822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761774.083833:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761774.083836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1223 00000100:00080000:3.0:1789761774.083842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761774.083859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761774.083866:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761774.083878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048786048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761774.083889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761774.083892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761774.083895:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761774.083901:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761774/real 1789761774] req@ffff9811e1a20000 x1876700048785664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761774.083901:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1223 00000100:00080000:3.0:1789761774.083908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761774.083916:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761774.083923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761774.083934:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048785664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761774.083936:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048786176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761774.083968:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761774.083973:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761774.083977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048785664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761774.083998:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048785792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761774.084018:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761774.084027:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761774/real 1789761774] req@ffff9811e1a20700 x1876700048785792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761774.084035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048785792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761774.084041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761774.084043:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761774.084046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048785792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761774.084055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761774.084069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761774.084076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761774/real 1789761774] req@ffff9811e1a21f80 x1876700048785920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761774.084084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048785920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761774.084089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761774.084091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761774.084093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048785920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761774.084096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761774.084107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761774.084113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761774/real 1789761774] req@ffff9811e1a20a80 x1876700048786048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761774.084120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048786048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761774.084125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761774.084126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761774.084128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761774.084131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761774.084148:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761774.084155:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761774/real 1789761774] req@ffff9811e1a20e00 x1876700048786176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761774.084170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048786176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761774.084175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761774.084177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761774.084179:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761774.339558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761774.979674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761775.363592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761776.003660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761776.387606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761777.027675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761777.411588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761778.051632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761778.435601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761779.075620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761779.203610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761779.203623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761779.203627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761779.203640:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1228 00000100:00080000:0.0:1789761779.203654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761779.203662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761779.203711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048786304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761779.203740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761779.203744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761779.203746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761779.203750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1228 00000100:00080000:0.0:1789761779.203753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761779.203757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761779.203768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761779.203781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048786432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761779.203793:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048786304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761779.203794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761779.203841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761779.203843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761779.203855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1228 00000100:00080000:0.0:1789761779.203860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761779.203865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761779.203870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761779.203884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048786560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761779.203890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761779.203901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761779.203904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761779.203907:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761779.203911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1228 00000100:00080000:0.0:1789761779.203916:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761779.203920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761779.203926:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761779.203937:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048786688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761779.203946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761779.203950:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761779/real 1789761779] req@ffff9811c2e29180 x1876700048786304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761779.203952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761779.203956:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761779.203964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1228 00000100:00080000:0.0:1789761779.203969:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761779.203973:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761779.203977:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048786304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761779.203988:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761779.203990:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761779.203996:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761779.203999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048786304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761779.204004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048786816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761779.204025:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048786432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761779.204051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761779.204060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761779/real 1789761779] req@ffff9811c2e2aa00 x1876700048786432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761779.204068:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048786432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761779.204082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761779.204084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761779.204087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048786432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761779.204090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761779.204104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761779.204110:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761779/real 1789761779] req@ffff9811c2e2b100 x1876700048786560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761779.204122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048786560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761779.204127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761779.204129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761779.204131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761779.204134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761779.204145:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761779.204150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761779/real 1789761779] req@ffff9811c2e29880 x1876700048786688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761779.204156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048786688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761779.204165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761779.204166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761779.204173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761779.204177:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761779.204190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761779.204196:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761779/real 1789761779] req@ffff9811c2e2ad80 x1876700048786816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761779.204203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048786816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761779.204208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761779.204209:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761779.204211:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048786816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761779.459555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761780.099577:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761780.483585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761781.123610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761781.507583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761782.147594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761782.531596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761783.171634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761783.555575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761784.195624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761784.323588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761784.323603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761784.323607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761784.323618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1233 00000100:00080000:0.0:1789761784.323631:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761784.323641:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761784.323689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048786944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761784.323715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761784.323718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761784.323719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761784.323722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1233 00000100:00080000:0.0:1789761784.323725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761784.323727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761784.323735:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761784.323745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048787072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761784.323753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761784.323756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761784.323757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761784.323759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1233 00000100:00100000:2.0:1789761784.323782:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048786944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761784.323796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761784.323802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761784.323808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761784.323821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048787200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761784.323832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761784.323835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761784.323836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761784.323839:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1233 00000100:00080000:0.0:1789761784.323845:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761784.323848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761784.323852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761784.323860:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761784.323862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048787328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761784.323870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761784.323873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761784.323874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761784.323876:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1233 00000100:00080000:0.0:1789761784.323878:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761784.323881:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761784.323883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761784.323889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048787456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761784.323913:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761784/real 1789761784] req@ffff9811c2e29c00 x1876700048786944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761784.323929:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048786944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761784.323937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761784.323940:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761784.323943:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048786944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761784.323949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048787072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761784.323966:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761784.323981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761784/real 1789761784] req@ffff9811c2e2a680 x1876700048787072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761784.323989:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048787072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761784.323994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761784.323997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761784.323999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048787072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761784.324019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761784.324030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761784.324036:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761784/real 1789761784] req@ffff9811c2e28700 x1876700048787200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761784.324046:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048787200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761784.324053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761784.324056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761784.324060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761784.324065:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761784.324094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761784.324105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761784/real 1789761784] req@ffff9811c2e2ad80 x1876700048787328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761784.324116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048787328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761784.324130:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761784.324133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761784.324135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761784.324139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761784.324153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761784.324159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761784/real 1789761784] req@ffff9811c2e29880 x1876700048787456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761784.324166:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048787456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761784.324171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761784.324173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761784.324175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761784.579586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761785.219583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761785.603551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761786.243617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761786.627540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761787.267583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761787.651536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761788.291631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761788.675570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761789.315609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761789.443595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761789.443605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761789.443609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761789.443620:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1239 00000100:00080000:3.0:1789761789.443631:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761789.443638:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761789.443678:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048787584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761789.443698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761789.443715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761789.443716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761789.443719:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1239 00000100:00080000:3.0:1789761789.443723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761789.443726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761789.443745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761789.443760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048787712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761789.443771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761789.443775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761789.443781:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761789.443783:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048787584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761789.443791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1239 00000100:00080000:3.0:1789761789.443797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761789.443801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761789.443809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761789.443828:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048787840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761789.443840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761789.443844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761789.443847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761789.443852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761789.443854:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1239 00000100:00080000:3.0:1789761789.443862:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761789.443867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761789.443882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761789.443890:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761789/real 1789761789] req@ffff9811e1a21500 x1876700048787584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761789.443901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048787968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761789.443910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048787584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761789.443926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761789.443933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761789.443935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761789.443935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761789.443947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1239 00000100:00080000:2.0:1789761789.443948:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761789.443964:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761789.443965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048787584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761789.443970:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761789.443975:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761789.443990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048788096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761789.444009:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048787712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761789.444031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761789.444040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761789/real 1789761789] req@ffff9811e1a20380 x1876700048787712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761789.444047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048787712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761789.444052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761789.444054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761789.444056:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048787712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761789.444059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761789.444068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761789.444073:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761789/real 1789761789] req@ffff9811e1a20e00 x1876700048787840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761789.444079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048787840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761789.444083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761789.444085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761789.444086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761789.444089:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761789.444098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761789.444103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761789/real 1789761789] req@ffff9811e1a20a80 x1876700048787968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761789.444114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048787968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761789.444118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761789.444120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761789.444122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048787968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761789.444132:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761789.444144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761789.444150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761789/real 1789761789] req@ffff9811e1a21f80 x1876700048788096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761789.444156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048788096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761789.444161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761789.444163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761789.444164:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761789.699579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761790.339622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761790.723554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761791.363634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761791.747579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761792.387625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761792.771600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761793.411641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761793.795592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761794.435650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761794.563546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761794.563557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761794.563562:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761794.563572:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1244 00000100:00080000:0.0:1789761794.563585:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761794.563598:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761794.563644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048788224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761794.563667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761794.563670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761794.563672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761794.563674:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1244 00000100:00080000:0.0:1789761794.563677:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761794.563679:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761794.563689:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761794.563712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048788352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761794.563722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761794.563725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761794.563728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761794.563732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1244 00000100:00080000:0.0:1789761794.563735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761794.563740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761794.563750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048788224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789761794.563764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761794.563777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048788480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761794.563787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761794.563790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761794.563793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761794.563796:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1244 00000100:00080000:0.0:1789761794.563801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761794.563804:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761794.563807:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761794.563816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048788608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761794.563822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761794.563825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761794.563826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761794.563829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1244 00000100:00080000:0.0:1789761794.563831:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761794.563833:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761794.563836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761794.563848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048788736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761794.563877:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048788352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761794.563918:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761794.563957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761794.563985:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761794.819579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761795.459607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761795.843612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761796.483738:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761796.867545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761797.507639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761797.635779:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789761797.635833:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761797.635853:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761797.635898:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761797.635912:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761797.635922:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761797.635934:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761797.635946:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761794/real 1789761797] req@ffff9811c2e29180 x1876700048788480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761797.635980:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048788480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761797.635994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761797.635999:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761797.636004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761797.636012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761794/real 1789761797] req@ffff9811c2e2bb80 x1876700048788608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761797.636025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048788608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761797.636033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761797.636037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761797.636040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761797.636064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761794/real 1789761797] req@ffff9811c2e28e00 x1876700048788736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761797.636076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048788736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761797.636085:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761797.636089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761797.636093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048788736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761797.636140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761794/real 1789761797] req@ffff9811c2e2b100 x1876700048788224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761797.636158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048788224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761797.636168:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761797.636188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761797.636192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048788224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761797.636200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761794/real 1789761797] req@ffff9811c2e2aa00 x1876700048788352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761797.636215:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048788352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761797.636222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761797.636224:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761797.636227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048788352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761797.891606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761798.531701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761798.915541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761799.555606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761799.683569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761799.683580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761799.683604:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761799.683615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1249 00000100:00080000:0.0:1789761799.683628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761799.683636:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761799.683700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048788864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761799.683736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761799.683740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761799.683743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761799.683748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1249 00000100:00080000:0.0:1789761799.683753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761799.683758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761799.683771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761799.683793:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048788864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761799.683798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048788992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761799.683816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761799.683820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761799.683822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761799.683826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1249 00000100:00080000:0.0:1789761799.683831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761799.683836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761799.683842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761799.683858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048789120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761799.683899:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761799.683906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761799.683911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761799.683914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761799.683921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1249 00000100:00080000:0.0:1789761799.683929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761799.683934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761799.683941:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761799.683962:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048789248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761799.683962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761799/real 1789761799] req@ffff9811c2e28380 x1876700048788864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761799.683989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761799.683995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761799.683997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761799.684004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1249 00000100:00100000:2.0:1789761799.684005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048788864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761799.684011:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761799.684021:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789761799.684028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761799.684029:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761799.684044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048789376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761799.684046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761799.684056:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048788864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761799.684094:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048788992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761799.684127:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761799.684142:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761799/real 1789761799] req@ffff9811c2e28a80 x1876700048788992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761799.684156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048788992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761799.684165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761799.684168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761799.684178:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048788992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761799.684185:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761799.684207:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761799.684219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761799/real 1789761799] req@ffff9811c2e29f80 x1876700048789120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761799.684232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048789120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761799.684241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761799.684244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761799.684248:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761799.684255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761799.684290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761799.684304:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761799/real 1789761799] req@ffff9811c2e2b800 x1876700048789248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761799.684317:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048789248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761799.684325:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761799.684329:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761799.684332:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761799.684339:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761799.684358:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761799.684369:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761799/real 1789761799] req@ffff9811c2e2b480 x1876700048789376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761799.684380:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048789376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761799.684388:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761799.684391:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761799.684395:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761799.939609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761800.579633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761800.963558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761801.603669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761801.987531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761802.627628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761803.011550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761803.651680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761804.035533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761804.675610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761804.803636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761804.803672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761804.803677:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761804.803691:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1254 00000100:00080000:3.0:1789761804.803706:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761804.803716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761804.803765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048789504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761804.803797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761804.803801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761804.803803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761804.803808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1254 00000100:00080000:3.0:1789761804.803812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761804.803818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761804.803830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761804.803846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048789632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761804.803856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048789504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761804.803869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761804.803874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761804.803876:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761804.803883:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1254 00000100:00080000:3.0:1789761804.803887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761804.803892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761804.803898:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761804.803912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048789760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761804.803932:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761804.803941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761804.803946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761804.803949:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761804.803956:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1254 00000100:00080000:3.0:1789761804.803962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761804.803966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761804.803974:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761804.803980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761804/real 1789761804] req@ffff9811e1a20700 x1876700048789504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761804.803990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048789888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761804.804002:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761804.804015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761804.804019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761804.804026:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1254 00000100:00080000:3.0:1789761804.804032:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761804.804038:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761804.804045:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048789504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761804.804045:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761804.804068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761804.804069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048790016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761804.804078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761804.804086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048789504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761804.804125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048789632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761804.804149:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761804.804163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761804/real 1789761804] req@ffff9811e1a20000 x1876700048789632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761804.804172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048789632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761804.804181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761804.804185:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761804.804188:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048789632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761804.804194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761804.804216:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761804.804228:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761804/real 1789761804] req@ffff9811e1a23480 x1876700048789760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761804.804237:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048789760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761804.804252:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761804.804255:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761804.804257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761804.804262:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761804.804280:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761804.804291:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761804/real 1789761804] req@ffff9811e1a23800 x1876700048789888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761804.804304:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048789888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761804.804312:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761804.804316:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761804.804319:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048789888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761804.804326:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761804.804342:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761804.804349:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761804/real 1789761804] req@ffff9811e1a23100 x1876700048790016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761804.804357:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048790016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761804.804362:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761804.804364:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761804.804371:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761805.059544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761805.699647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761806.083542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761806.723590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761807.107584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761807.747655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761808.131553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761808.771618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761809.155546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761809.795622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761809.923628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761809.923644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761809.923649:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761809.923660:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1259 00000100:00080000:2.0:1789761809.923672:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761809.923680:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761809.923724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048790144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761809.923751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761809.923754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761809.923756:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761809.923759:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1259 00000100:00080000:2.0:1789761809.923761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761809.923763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761809.923772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761809.923782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048790272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761809.923793:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048790144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761809.923793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761809.923801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761809.923804:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761809.923807:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1259 00000100:00080000:2.0:1789761809.923810:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761809.923834:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761809.923839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761809.923848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048790400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761809.923856:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761809.923859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761809.923860:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761809.923862:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1259 00000100:00080000:2.0:1789761809.923865:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761809.923869:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761809.923870:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761809.923877:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761809.923885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048790528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761809.923892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761809.923895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761809.923897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761809.923900:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1259 00000100:00080000:2.0:1789761809.923903:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761809.923905:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761809.923908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761809.923915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048790656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761809.923929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761809/real 1789761809] req@ffff9811d8639180 x1876700048790144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761809.923959:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048790144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761809.923968:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761809.923971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761809.923974:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048790144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761809.923997:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048790272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761809.924015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761809.924025:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761809/real 1789761809] req@ffff9811d863bb80 x1876700048790272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761809.924033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048790272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761809.924039:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761809.924041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761809.924044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048790272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761809.924047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761809.924061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761809.924072:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761809/real 1789761809] req@ffff9811d863b100 x1876700048790400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761809.924084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048790400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761809.924106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761809.924111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761809.924114:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761809.924119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761809.924137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761809.924145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761809/real 1789761809] req@ffff9811d8639c00 x1876700048790528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761809.924153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048790528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761809.924159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761809.924161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761809.924163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761809.924166:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761809.924176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761809.924183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761809/real 1789761809] req@ffff9811d863ad80 x1876700048790656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761809.924195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048790656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761809.924204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761809.924207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761809.924218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048790656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761810.179568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761810.819671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761811.203563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761811.843562:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761812.227537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761812.867583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761813.251641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761813.891661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761814.275585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761814.915616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761815.043611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761815.043647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761815.043651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761815.043662:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1264 00000100:00080000:2.0:1789761815.043674:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761815.043681:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761815.043720:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048790784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761815.043747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761815.043750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761815.043752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761815.043755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1264 00000100:00080000:2.0:1789761815.043757:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761815.043760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761815.043768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761815.043779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048790912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761815.043788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761815.043791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761815.043793:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761815.043799:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1264 00000100:00100000:0.0:1789761815.043801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048790784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761815.043801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761815.043807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761815.043813:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761815.043819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048791040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761815.043833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761815.043836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761815.043837:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761815.043841:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1264 00000100:00080000:2.0:1789761815.043844:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761815.043847:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761815.043851:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761815.043859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048791168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761815.043866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761815.043869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761815.043870:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761815.043872:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1264 00000100:00080000:2.0:1789761815.043874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761815.043877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761815.043882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789761815.043884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761815.043893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048791296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761815.043938:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761815/real 1789761815] req@ffff9811d8639880 x1876700048790784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761815.043963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048790784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761815.043975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761815.043980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761815.043985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048790784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761815.043993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048790912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761815.044019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761815.044034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761815/real 1789761815] req@ffff9811d8639500 x1876700048790912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761815.044043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048790912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761815.044049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761815.044052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761815.044054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048790912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761815.044078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761815.044093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761815.044109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761815/real 1789761815] req@ffff9811d8639f80 x1876700048791040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761815.044117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048791040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761815.044123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761815.044125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761815.044127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761815.044131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761815.044143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761815.044149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761815/real 1789761815] req@ffff9811d863a680 x1876700048791168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761815.044156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048791168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761815.044161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761815.044163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761815.044165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761815.044168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761815.044178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761815.044183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761815/real 1789761815] req@ffff9811d8638a80 x1876700048791296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761815.044189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048791296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761815.044202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761815.044206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761815.044209:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761815.299597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761815.939729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761816.323608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761816.963588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761817.347577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761817.987623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761818.371568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761819.011624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761819.395621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761820.035655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761820.163590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761820.163599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761820.163605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761820.163618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1269 00000100:00080000:2.0:1789761820.163644:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761820.163650:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761820.163681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048791424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761820.163704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761820.163706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761820.163708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761820.163711:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1269 00000100:00080000:2.0:1789761820.163713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761820.163716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761820.163725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761820.163734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048791552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761820.163746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761820.163748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048791424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761820.163750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761820.163755:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761820.163767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1269 00000100:00080000:2.0:1789761820.163770:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761820.163772:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761820.163775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761820.163783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048791680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761820.163790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761820.163798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761820.163799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761820.163803:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1269 00000100:00080000:2.0:1789761820.163806:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761820.163809:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761820.163813:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761820.163820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048791808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761820.163829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761820.163832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761820.163833:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761820.163835:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1269 00000100:00080000:2.0:1789761820.163838:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761820.163840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761820.163843:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761820.163853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048791936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761820.163863:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761820.163916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761820/real 1789761820] req@ffff9811d863aa00 x1876700048791424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761820.163933:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048791424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761820.163943:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761820.163946:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761820.163949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048791424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761820.163973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048791552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761820.163992:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761820.164001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761820/real 1789761820] req@ffff9811d8638000 x1876700048791552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761820.164010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048791552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761820.164016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761820.164019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761820.164021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048791552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761820.164035:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761820.164051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761820.164058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761820/real 1789761820] req@ffff9811d863b800 x1876700048791680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761820.164067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048791680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761820.164072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761820.164074:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761820.164076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761820.164079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761820.164091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761820.164097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761820/real 1789761820] req@ffff9811d863b480 x1876700048791808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761820.164113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048791808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761820.164119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761820.164121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761820.164124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761820.164130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761820.164151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761820.164175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761820/real 1789761820] req@ffff9811d8638380 x1876700048791936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761820.164197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048791936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761820.164203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761820.164205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761820.164207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048791936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761820.419589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761821.059619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761821.443621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761822.083631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761822.467606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761823.107664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761823.491619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761824.131622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761824.515554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761825.155670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761825.283604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761825.283615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761825.283620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761825.283632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1274 00000100:00080000:3.0:1789761825.283644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761825.283653:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761825.283699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048792064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761825.283722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761825.283725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761825.283726:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761825.283729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1274 00000100:00080000:3.0:1789761825.283732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761825.283734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761825.283742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761825.283751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048792192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761825.283768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761825.283793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761825.283796:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048792064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761825.283797:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761825.283811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1274 00000100:00080000:3.0:1789761825.283814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761825.283819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761825.283823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761825.283839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048792320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761825.283847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761825.283850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761825.283852:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761825.283856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1274 00000100:00080000:3.0:1789761825.283859:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761825.283861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761825.283864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761825.283871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048792448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761825.283877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761825.283879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761825.283882:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761825.283888:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761825.283892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1274 00000100:00080000:3.0:1789761825.283900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761825.283903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761825.283913:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761825.283926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048792576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761825.283948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761825/real 1789761825] req@ffff9811e1a22300 x1876700048792064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761825.283972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048792064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761825.283985:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761825.283989:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761825.283994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048792064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761825.284002:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048792192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761825.284022:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761825.284032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761825/real 1789761825] req@ffff9811e1a21180 x1876700048792192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761825.284059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048792192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761825.284065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761825.284067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761825.284069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048792192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761825.284089:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761825.284103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761825.284110:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761825/real 1789761825] req@ffff9811e1a22a00 x1876700048792320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761825.284118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048792320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761825.284123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761825.284125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761825.284127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761825.284131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761825.284141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761825.284147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761825/real 1789761825] req@ffff9811e1a21c00 x1876700048792448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761825.284164:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048792448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761825.284179:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761825.284181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761825.284183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761825.284188:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761825.284206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761825.284213:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761825/real 1789761825] req@ffff9811e1a21880 x1876700048792576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761825.284221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048792576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761825.284226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761825.284228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761825.284231:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761825.539582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761826.179633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761826.563556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761827.203659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761827.587570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761828.227599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761828.611585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761829.251638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761829.635566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761830.275612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761830.403555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761830.403563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761830.403567:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761830.403575:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1279 00000100:00080000:3.0:1789761830.403584:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761830.403591:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761830.403632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048792704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761830.403662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761830.403667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761830.403670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761830.403675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1279 00000100:00080000:3.0:1789761830.403680:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761830.403684:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761830.403694:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761830.403698:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048792704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761830.403712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048792832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761830.403725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761830.403739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761830.403742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761830.403747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1279 00000100:00080000:3.0:1789761830.403753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761830.403759:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761830.403760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761830.403774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761830.403796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048792960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761830.403809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761830.403813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789761830.403818:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761830/real 1789761830] req@ffff9811e1a22d80 x1876700048792704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761830.403819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761830.403830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1279 00000100:00100000:0.0:1789761830.403853:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048792704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761830.403866:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761830.403880:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761830.403885:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048792704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761830.403916:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048792832:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761830.403917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761830.403924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761830.403930:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761830.403943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048793088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761830.403952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761830.403952:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761830.403958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761830.403960:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761830.403964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1279 00000100:00080000:3.0:1789761830.403967:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761830.403970:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761830.403974:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761830/real 1789761830] req@ffff9811e1a22680 x1876700048792832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761830.403982:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761830.403991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048793216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761830.404027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048792832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761830.404037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761830.404042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761830.404046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048792832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761830.404053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761830.404075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761830.404088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761830/real 1789761830] req@ffff9811e1a21880 x1876700048792960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761830.404098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048792960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761830.404104:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761830.404106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761830.404109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048792960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761830.404120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761830.404143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761830.404153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761830/real 1789761830] req@ffff9811e1a21c00 x1876700048793088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761830.404162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048793088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761830.404168:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761830.404171:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761830.404173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761830.404177:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761830.404187:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761830.404194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761830/real 1789761830] req@ffff9811e1a22a00 x1876700048793216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761830.404202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048793216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761830.404207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761830.404210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761830.404212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761830.659576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761831.299654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761831.683554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761832.323596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761832.707586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761833.347583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761833.731539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761834.371606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761834.755549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761835.395607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761835.523574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761835.523584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761835.523611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761835.523622:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1285 00000100:00080000:0.0:1789761835.523634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761835.523641:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761835.523681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048793344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761835.523718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761835.523722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761835.523724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761835.523728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1285 00000100:00080000:0.0:1789761835.523732:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761835.523736:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761835.523748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761835.523764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048793344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761835.523765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048793472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761835.523782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761835.523785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761835.523786:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761835.523789:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1285 00000100:00080000:0.0:1789761835.523792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761835.523795:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761835.523799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761835.523808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048793600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761835.523833:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761835.523837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761835.523840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761835.523844:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1285 00000800:00000100:2.0:1789761835.523856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761835.523858:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761835.523866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761835.523873:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761835.523888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048793728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761835.523899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761835.523902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761835.523905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761835.523909:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1285 00000100:00100000:2.0:1789761835.523914:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761835/real 1789761835] req@ffff9811c2e29500 x1876700048793344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761835.523916:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761835.523924:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761835.523932:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761835.523941:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048793344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761835.523944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048793856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761835.523975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761835.523980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761835.523985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048793344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761835.524008:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048793472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761835.524028:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761835.524038:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761835/real 1789761835] req@ffff9811c2e28000 x1876700048793472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761835.524047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048793472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761835.524053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761835.524056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761835.524058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048793472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761835.524062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761835.524073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761835.524079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761835/real 1789761835] req@ffff9811c2e2a300 x1876700048793600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761835.524087:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048793600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761835.524092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761835.524094:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761835.524096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761835.524105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761835.524120:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761835.524126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761835/real 1789761835] req@ffff9811c2e2b480 x1876700048793728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761835.524135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048793728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761835.524140:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761835.524142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761835.524144:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761835.524148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761835.524159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761835.524164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761835/real 1789761835] req@ffff9811c2e2b800 x1876700048793856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761835.524172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048793856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761835.524177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761835.524178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761835.524180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048793856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761835.779535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761836.419599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761836.803566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761837.443686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761837.827577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761838.467564:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761838.851611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761839.491713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761839.875588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761840.515672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761840.643590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761840.643601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761840.643605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761840.643617:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1290 00000100:00080000:3.0:1789761840.643629:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761840.643637:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761840.643679:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048793984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761840.643708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761840.643711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761840.643712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761840.643716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1290 00000100:00080000:3.0:1789761840.643718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761840.643721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761840.643730:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761840.643739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048794112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761840.643749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761840.643752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761840.643753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761840.643758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1290 00000100:00100000:2.0:1789761840.643767:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048793984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761840.643779:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761840.643786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761840.643790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761840.643800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048794240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761840.643808:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761840.643811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761840.643812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761840.643815:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1290 00000100:00080000:3.0:1789761840.643818:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761840.643820:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761840.643823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761840.643831:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048794368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761840.643837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761840.643839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761840.643841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761840.643843:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1290 00000100:00080000:3.0:1789761840.643846:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761840.643848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761840.643851:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761840.643858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048794496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761840.643864:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761840.643929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761840/real 1789761840] req@ffff9811e1a21180 x1876700048793984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761840.643955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048793984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761840.643968:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761840.643972:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761840.643975:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048793984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761840.643981:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048794112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761840.644001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761840.644011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761840/real 1789761840] req@ffff9811e1a22300 x1876700048794112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761840.644020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048794112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761840.644026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761840.644028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761840.644031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048794112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761840.644052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761840.644064:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761840.644070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761840/real 1789761840] req@ffff9811e1a23100 x1876700048794240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761840.644088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048794240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761840.644094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761840.644097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761840.644099:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761840.644102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761840.644115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761840.644122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761840/real 1789761840] req@ffff9811e1a23800 x1876700048794368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761840.644129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048794368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761840.644134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761840.644136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761840.644138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761840.644142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761840.644152:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761840.644157:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761840/real 1789761840] req@ffff9811e1a23480 x1876700048794496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761840.644164:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048794496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761840.644179:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761840.644181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761840.644183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761840.899586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761841.539678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761841.923569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761842.563651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761842.947582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761843.587604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761843.971559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761844.611615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761844.995566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761845.635579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761845.763569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761845.763582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761845.763587:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761845.763600:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1295 00000100:00080000:1.0:1789761845.763614:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761845.763624:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761845.763671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048794624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761845.763713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761845.763717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761845.763719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761845.763724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1295 00000100:00080000:1.0:1789761845.763729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761845.763753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761845.763765:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761845.763781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048794752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761845.763793:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048794624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761845.763796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761845.763803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761845.763805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761845.763810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1295 00000100:00080000:1.0:1789761845.763815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761845.763820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761845.763826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761845.763843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048794880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761845.763855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761845.763859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761845.763861:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761845.763866:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1295 00000800:00000100:2.0:1789761845.763870:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761845.763873:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761845.763881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761845.763887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761845.763900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048795008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761845.763911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761845.763921:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761845.763924:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761845.763930:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761845/real 1789761845] req@ffff9811c92c5c00 x1876700048794624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761845.763931:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1295 00000100:00080000:1.0:1789761845.763938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761845.763947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761845.763956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761845.763964:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048794624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761845.763972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048795136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761845.763983:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761845.763992:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761845.763999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048794624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761845.764033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048794752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761845.764054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761845.764065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761845/real 1789761845] req@ffff9811c92c6680 x1876700048794752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761845.764075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048794752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761845.764092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761845.764096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761845.764101:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048794752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761845.764107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761845.764141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761845.764150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761845/real 1789761845] req@ffff9811c92c4000 x1876700048794880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761845.764158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048794880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761845.764164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761845.764167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761845.764169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048794880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761845.764173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761845.764190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761845.764200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761845/real 1789761845] req@ffff9811c92c4e00 x1876700048795008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761845.764213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048795008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761845.764222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761845.764226:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761845.764235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761845.764240:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761845.764265:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761845.764274:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761845/real 1789761845] req@ffff9811c92c4380 x1876700048795136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761845.764282:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048795136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761845.764287:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761845.764290:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761845.764292:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761846.019557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761846.659615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761847.043542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761847.683626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761848.067584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761848.707598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761849.091555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761849.731588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761850.115569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761850.755605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761850.883599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761850.883607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761850.883637:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761850.883645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1300 00000100:00080000:0.0:1789761850.883655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761850.883661:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761850.883707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048795264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761850.883728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761850.883731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761850.883732:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761850.883735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1300 00000100:00080000:0.0:1789761850.883737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761850.883740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761850.883747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761850.883755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048795392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761850.883765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761850.883767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761850.883771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761850.883772:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048795264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761850.883776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1300 00000100:00080000:0.0:1789761850.883782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761850.883785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761850.883788:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761850.883796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048795520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761850.883812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761850.883815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761850.883816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761850.883818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1300 00000100:00080000:0.0:1789761850.883822:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761850.883825:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761850.883829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761850.883838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048795648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761850.883844:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761850.883851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761850.883854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761850.883856:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761850.883859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1300 00000100:00080000:0.0:1789761850.883862:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761850.883864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761850.883867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761850.883874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048795776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761850.883894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761850/real 1789761850] req@ffff9811c2e29f80 x1876700048795264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761850.883910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048795264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761850.883927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761850.883929:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761850.883949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048795264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761850.883967:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048795392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761850.883991:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761850.884005:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761850/real 1789761850] req@ffff9811c2e28a80 x1876700048795392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761850.884018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048795392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761850.884024:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761850.884026:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761850.884028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048795392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761850.884032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761850.884047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761850.884054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761850/real 1789761850] req@ffff9811c2e28380 x1876700048795520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761850.884061:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048795520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761850.884067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761850.884069:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761850.884071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761850.884074:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761850.884084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761850.884097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761850/real 1789761850] req@ffff9811c2e2aa00 x1876700048795648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761850.884105:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048795648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761850.884110:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761850.884112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761850.884114:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761850.884118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761850.884131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761850.884141:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761850/real 1789761850] req@ffff9811c2e2b100 x1876700048795776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761850.884171:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048795776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761850.884181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761850.884185:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761850.884189:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048795776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761851.139526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761851.779596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761852.163687:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761852.803653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761853.187528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761853.827623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761854.211616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761854.851618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761855.235684:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761855.875569:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761856.003571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761856.003579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761856.003582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761856.003591:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1305 00000100:00080000:3.0:1789761856.003601:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761856.003607:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761856.003639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048795904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761856.003661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761856.003664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761856.003665:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761856.003668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1305 00000100:00080000:3.0:1789761856.003670:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761856.003673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761856.003680:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761856.003689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048796032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761856.003697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761856.003700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761856.003701:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761856.003704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1305 00000100:00100000:2.0:1789761856.003730:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048795904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761856.003733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761856.003741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761856.003745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761856.003757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048796160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761856.003764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761856.003767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761856.003769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761856.003771:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1305 00000100:00080000:3.0:1789761856.003787:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761856.003790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761856.003793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761856.003804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048796288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761856.003806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761856.003816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761856.003821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761856.003823:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761856.003828:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1305 00000100:00080000:3.0:1789761856.003832:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761856.003837:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761856.003843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761856.003853:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761856/real 1789761856] req@ffff9811e1a20000 x1876700048795904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761856.003858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048796416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761856.003913:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048795904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761856.003926:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761856.003931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761856.003935:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048795904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761856.003944:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048796032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761856.003980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761856.003994:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761856/real 1789761856] req@ffff9811e1a20700 x1876700048796032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761856.004008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048796032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761856.004016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761856.004019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761856.004022:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048796032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761856.004057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761856.004079:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761856.004091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761856/real 1789761856] req@ffff9811e1a21f80 x1876700048796160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761856.004113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048796160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761856.004134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761856.004138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761856.004142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761856.004148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761856.004172:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761856.004185:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761856/real 1789761856] req@ffff9811e1a20a80 x1876700048796288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761856.004197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048796288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761856.004206:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761856.004209:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761856.004213:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761856.004219:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761856.004257:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761856.004279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761856/real 1789761856] req@ffff9811e1a20e00 x1876700048796416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761856.004293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048796416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761856.004302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761856.004305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761856.004316:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761856.259602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761856.899600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761857.283581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761857.923589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761858.307616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761858.947637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761859.331547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761859.971585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761860.355530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761860.995649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761861.123605:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761861.123614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761861.123619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761861.123628:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1310 00000100:00080000:3.0:1789761861.123638:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761861.123646:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761861.123684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048796544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761861.123711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761861.123716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761861.123720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761861.123727:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1310 00000100:00080000:3.0:1789761861.123733:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761861.123737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761861.123746:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761861.123747:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048796544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761861.123759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048796672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761861.123770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761861.123773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761861.123776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761861.123780:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1310 00000100:00080000:3.0:1789761861.123798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761861.123803:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761861.123809:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761861.123824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048796800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761861.123832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761861.123835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761861.123836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761861.123840:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1310 00000100:00080000:3.0:1789761861.123845:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761861.123848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761861.123852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761861.123859:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048796672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761861.123865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048796928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761861.123874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761861.123876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761861.123878:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761861.123881:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1310 00000100:00080000:3.0:1789761861.123891:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761861.123894:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761861.123899:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761861.123903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761861.123911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048797056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761861.123925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761861.123950:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761861.379532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761862.019672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761862.403565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761863.043619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761863.427563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761864.067747:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761864.195755:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789761864.195818:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761864.195833:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761864.195900:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761864.195917:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761864.195923:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789761864.195928:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761864.195999:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761861/real 1789761864] req@ffff9811e1a21f80 x1876700048797056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761864.196035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048797056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761864.196066:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761864.196071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761864.196076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761864.196084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761861/real 1789761864] req@ffff9811e1a20a80 x1876700048796928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761864.196097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048796928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761864.196123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761864.196137:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761864.196141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761864.196149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761861/real 1789761864] req@ffff9811e1a21500 x1876700048796672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761864.196165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048796672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761864.196191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761864.196196:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761864.196201:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048796672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761864.196208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761861/real 1789761864] req@ffff9811e1a20e00 x1876700048796800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761864.196222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048796800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761864.196231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761864.196234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761864.196238:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048796800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761864.196246:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761861/real 1789761864] req@ffff9811e1a20380 x1876700048796544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761864.196261:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048796544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761864.196279:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761864.196283:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761864.196288:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048796544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761864.451626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761865.091565:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761865.475577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761866.115679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761866.243564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761866.243589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761866.243594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761866.243607:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1315 00000100:00080000:3.0:1789761866.243622:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761866.243662:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761866.243748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048797184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761866.243792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761866.243797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761866.243799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761866.243804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1315 00000100:00080000:3.0:1789761866.243809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761866.243815:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761866.243827:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761866.243845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048797312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761866.243858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761866.243862:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761866.243864:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761866.243869:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1315 00000100:00080000:3.0:1789761866.243873:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761866.243877:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761866.243901:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761866.243917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048797440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761866.243933:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048797184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761866.243937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761866.243943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761866.243945:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761866.243955:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1315 00000100:00080000:3.0:1789761866.243963:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761866.243971:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761866.243979:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761866.243998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048797568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761866.244012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761866.244017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761866.244020:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761866.244026:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1315 00000100:00080000:3.0:1789761866.244033:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761866.244039:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761866.244047:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761866.244056:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761866.244069:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048797696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761866.244126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761866/real 1789761866] req@ffff9811e1a20700 x1876700048797184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761866.244175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048797184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761866.244192:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761866.244200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761866.244206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048797184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761866.244217:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048797312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761866.244255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761866.244276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761866/real 1789761866] req@ffff9811e1a20000 x1876700048797312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761866.244296:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048797312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761866.244317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761866.244322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761866.244328:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048797312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761866.244374:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761866.244407:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761866.244424:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761866/real 1789761866] req@ffff9811e1a23480 x1876700048797440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761866.244442:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048797440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761866.244455:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761866.244510:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761866.244517:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761866.244527:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761866.244557:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761866.244575:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761866/real 1789761866] req@ffff9811e1a23800 x1876700048797568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761866.244593:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048797568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761866.244605:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761866.244610:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761866.244615:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761866.244624:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761866.244651:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761866.244665:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761866/real 1789761866] req@ffff9811e1a23100 x1876700048797696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761866.244693:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048797696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761866.244706:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761866.244710:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761866.244715:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048797696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761866.499632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761867.139616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761867.523536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761868.163642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761868.547539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761869.187631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761869.571528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761870.211671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761870.595552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761871.235578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761871.363553:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761871.363564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761871.363588:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761871.363598:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1320 00000100:00080000:3.0:1789761871.363610:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761871.363618:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761871.363657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048797824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761871.363692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761871.363699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761871.363702:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761871.363708:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1320 00000100:00080000:3.0:1789761871.363713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761871.363722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761871.363737:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761871.363753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048797824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761871.363764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048797952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761871.363784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761871.363789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761871.363791:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761871.363796:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1320 00000100:00080000:3.0:1789761871.363800:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761871.363804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761871.363810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761871.363824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048798080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761871.363850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761871.363861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761871.363866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761871.363869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761871.363876:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1320 00000100:00080000:3.0:1789761871.363882:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761871.363887:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761871.363895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761871.363905:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761871/real 1789761871] req@ffff9811e1a22300 x1876700048797824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761871.363911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048798208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761871.363958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761871.363963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761871.363966:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761871.363971:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1320 00000100:00100000:0.0:1789761871.363976:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048797824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761871.363978:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761871.363987:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761871.363997:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761871.364006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761871.364014:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048798336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761871.364019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761871.364032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048797824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761871.364064:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048797952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761871.364101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761871.364114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761871/real 1789761871] req@ffff9811e1a21180 x1876700048797952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761871.364148:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048797952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761871.364178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761871.364183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761871.364187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048797952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761871.364194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761871.364223:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761871.364241:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761871/real 1789761871] req@ffff9811e1a22a00 x1876700048798080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761871.364262:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048798080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761871.364276:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761871.364281:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761871.364288:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761871.364298:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761871.364323:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761871.364335:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761871/real 1789761871] req@ffff9811e1a21c00 x1876700048798208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761871.364348:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048798208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761871.364357:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761871.364360:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761871.364371:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761871.364378:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761871.364399:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761871.364410:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761871/real 1789761871] req@ffff9811e1a21880 x1876700048798336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761871.364422:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048798336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761871.364429:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761871.364432:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761871.364435:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761871.619532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761872.259581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761872.643523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761873.283577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761873.667616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761874.307583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761874.691602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761875.331612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761875.715574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761876.355590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761876.483646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761876.483660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761876.483711:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761876.483726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1326 00000100:00080000:3.0:1789761876.483740:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761876.483766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761876.483825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048798464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761876.483862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761876.483868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761876.483871:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761876.483877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1326 00000100:00080000:3.0:1789761876.483892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761876.483899:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761876.483914:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761876.483930:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048798464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761876.483934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048798592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761876.483961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761876.483966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761876.483969:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761876.483974:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1326 00000100:00080000:3.0:1789761876.483978:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761876.483983:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761876.483989:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761876.484001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048798720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761876.484031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761876.484035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761876.484036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761876.484039:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761876.484043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1326 00000100:00080000:3.0:1789761876.484064:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761876.484068:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761876.484074:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761876.484104:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048798848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761876.484114:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761876.484118:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761876.484119:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761876.484124:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1326 00000100:00100000:0.0:1789761876.484125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761876/real 1789761876] req@ffff9811e1a22680 x1876700048798464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761876.484139:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761876.484144:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761876.484149:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761876.484160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048798464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761876.484160:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048798976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761876.484174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761876.484179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761876.484184:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048798464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761876.484214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048798592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761876.484261:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761876.484285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761876/real 1789761876] req@ffff9811e1a22d80 x1876700048798592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761876.484298:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048798592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761876.484307:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761876.484310:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761876.484313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048798592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761876.484319:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761876.484337:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761876.484346:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761876/real 1789761876] req@ffff9811e1a21880 x1876700048798720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761876.484358:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048798720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761876.484365:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761876.484367:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761876.484370:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761876.484375:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761876.484390:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761876.484398:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761876/real 1789761876] req@ffff9811e1a21c00 x1876700048798848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761876.484407:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048798848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761876.484440:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761876.484445:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761876.484449:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761876.484457:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761876.484501:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761876.484558:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761876/real 1789761876] req@ffff9811e1a22a00 x1876700048798976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761876.484577:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048798976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761876.484589:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761876.484593:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761876.484599:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048798976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761876.739600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761877.379595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761877.763544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761878.403581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761878.787610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761879.427568:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761879.811571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761880.451563:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761880.835554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761881.475555:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761881.603625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761881.603636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761881.603641:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761881.603653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1331 00000100:00080000:3.0:1789761881.603697:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761881.603708:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761881.603746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048799104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761881.603802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761881.603807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761881.603809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761881.603813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1331 00000100:00080000:3.0:1789761881.603817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761881.603821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761881.603831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761881.603845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048799232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761881.603850:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048799104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761881.603866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761881.603870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761881.603872:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761881.603892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1331 00000100:00080000:3.0:1789761881.603896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761881.603901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761881.603907:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761881.603920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048799360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761881.603931:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761881.603933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761881.603945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761881.603948:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761881.603953:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1331 00000100:00080000:3.0:1789761881.603958:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761881.603964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761881.603970:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761881.603981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761881/real 1789761881] req@ffff9811e1a21180 x1876700048799104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761881.603986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048799488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761881.604011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048799104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761881.604016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761881.604024:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761881.604041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761881.604042:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761881.604060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1331 00000100:00080000:3.0:1789761881.604066:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761881.604070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789761881.604075:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761881.604076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048799104:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761881.604085:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761881.604101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048799616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761881.604119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048799232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761881.604168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761881.604184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761881/real 1789761881] req@ffff9811e1a22300 x1876700048799232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761881.604199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048799232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761881.604208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761881.604211:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761881.604215:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048799232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761881.604222:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761881.604241:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761881.604253:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761881/real 1789761881] req@ffff9811e1a23100 x1876700048799360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761881.604265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048799360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761881.604274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761881.604277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761881.604281:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761881.604287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761881.604315:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761881.604327:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761881/real 1789761881] req@ffff9811e1a23800 x1876700048799488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761881.604340:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048799488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761881.604349:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761881.604351:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761881.604354:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761881.604371:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761881.604389:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761881.604399:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761881/real 1789761881] req@ffff9811e1a23480 x1876700048799616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761881.604412:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048799616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761881.604417:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761881.604419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761881.604421:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048799616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761881.859599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761882.499594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761882.883573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761883.523590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761883.907612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761884.547604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761884.931575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761885.571613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761885.955566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761886.595613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761886.723564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761886.723578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761886.723584:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761886.723598:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1336 00000100:00080000:3.0:1789761886.723635:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761886.723646:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761886.723695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048799744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761886.723735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761886.723740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761886.723742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761886.723748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1336 00000100:00080000:3.0:1789761886.723752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761886.723757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761886.723769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761886.723787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048799872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761886.723797:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048799744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761886.723804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761886.723810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761886.723812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761886.723817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1336 00000100:00080000:3.0:1789761886.723822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761886.723826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761886.723832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761886.723847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048800000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761886.723858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761886.723869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761886.723872:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761886.723877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1336 00000800:00000100:0.0:1789761886.723883:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761886.723883:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761886.723892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761886.723900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761886.723915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048800128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761886.723926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761886.723931:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761886/real 1789761886] req@ffff9811e1a20000 x1876700048799744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761886.723932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761886.723938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761886.723947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1336 00000100:00080000:3.0:1789761886.723953:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761886.723974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048799744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761886.723975:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761886.723987:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789761886.723989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761886.723995:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761886.723999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048799744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761886.724039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048799872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761886.724064:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761886.724075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761886/real 1789761886] req@ffff9811e1a20700 x1876700048799872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761886.724084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048799872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761886.724090:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761886.724093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761886.724095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048799872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761886.724104:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789761886.724111:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048800256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761886.724121:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761886.724132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761886/real 1789761886] req@ffff9811e1a20380 x1876700048800000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761886.724147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048800000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761886.724166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761886.724169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761886.724172:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761886.724176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761886.724193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761886.724200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761886/real 1789761886] req@ffff9811e1a20e00 x1876700048800128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761886.724208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048800128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761886.724214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761886.724216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761886.724218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761886.724229:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761886.724239:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761886.724245:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761886/real 1789761886] req@ffff9811e1a21500 x1876700048800256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761886.724252:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048800256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761886.724258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761886.724260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761886.724268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761886.979542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761887.619616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761888.003581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761888.643569:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761889.027586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761889.667613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761890.051554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761890.691568:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761891.075565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761891.715638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761891.843568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761891.843580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761891.843585:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761891.843596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1341 00000100:00080000:3.0:1789761891.843637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761891.843645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761891.843685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048800384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761891.843719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761891.843722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761891.843723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761891.843727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1341 00000100:00080000:3.0:1789761891.843729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761891.843732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761891.843742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761891.843751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048800512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761891.843761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761891.843764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761891.843766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761891.843769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1341 00000100:00080000:3.0:1789761891.843771:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761891.843775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761891.843782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761891.843784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048800384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761891.843795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048800640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761891.843805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761891.843822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761891.843824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761891.843830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1341 00000100:00080000:3.0:1789761891.843835:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761891.843839:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761891.843844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761891.843855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048800768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761891.843863:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761891.843876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761891.843879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761891.843881:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761891.843886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1341 00000100:00080000:3.0:1789761891.843890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761891.843893:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761891.843897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761891.843910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048800896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761891.843920:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761891/real 1789761891] req@ffff9811e1a20a80 x1876700048800384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761891.843937:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048800384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761891.843947:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761891.843951:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761891.843955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048800384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761891.843961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048800512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761891.843996:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761891.844006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761891/real 1789761891] req@ffff9811e1a21f80 x1876700048800512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761891.844014:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048800512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761891.844019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761891.844022:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761891.844024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048800512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761891.844042:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761891.844054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761891.844075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761891/real 1789761891] req@ffff9811e1a21500 x1876700048800640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761891.844083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048800640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761891.844088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761891.844091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761891.844093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761891.844096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761891.844108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761891.844114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761891/real 1789761891] req@ffff9811e1a20e00 x1876700048800768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761891.844121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048800768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761891.844126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761891.844128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761891.844129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761891.844133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761891.844142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761891.844147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761891/real 1789761891] req@ffff9811e1a20380 x1876700048800896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761891.844154:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048800896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761891.844163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761891.844166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761891.844168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048800896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761892.099573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761892.739648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761893.123618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761893.763685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761894.147631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761894.787605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761895.171633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761895.811698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789761896.195615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761896.835603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761896.963529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761896.963537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761896.963541:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761896.963548:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1346 00000100:00080000:3.0:1789761896.963574:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761896.963581:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761896.963610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048801024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761896.963636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761896.963639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761896.963641:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761896.963643:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1346 00000100:00080000:3.0:1789761896.963645:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761896.963647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761896.963654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761896.963664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048801152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761896.963672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761896.963675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761896.963676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761896.963679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1346 00000100:00080000:3.0:1789761896.963682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761896.963684:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761896.963690:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048801024:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789761896.963692:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761896.963707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048801280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761896.963719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761896.963729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761896.963732:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761896.963736:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1346 00000100:00080000:3.0:1789761896.963741:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761896.963745:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761896.963750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761896.963760:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761896.963761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048801408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761896.963770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761896.963772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761896.963774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761896.963777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1346 00000100:00080000:3.0:1789761896.963780:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761896.963783:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761896.963785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761896.963792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048801536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761896.963805:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761896/real 1789761896] req@ffff9811e1a20700 x1876700048801024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761896.963822:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048801024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761896.963831:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761896.963834:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761896.963837:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048801024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761896.963843:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048801152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761896.963868:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761896.963880:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761896/real 1789761896] req@ffff9811e1a20000 x1876700048801152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761896.963888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048801152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761896.963893:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761896.963896:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761896.963898:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048801152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761896.963922:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761896.963955:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761896.963964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761896/real 1789761896] req@ffff9811e1a23480 x1876700048801280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761896.963972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048801280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761896.963977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761896.963980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761896.963982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761896.963986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761896.963997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761896.964003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761896/real 1789761896] req@ffff9811e1a23800 x1876700048801408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761896.964010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048801408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761896.964015:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761896.964018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761896.964021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761896.964027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761896.964045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761896.964056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761896/real 1789761896] req@ffff9811e1a23100 x1876700048801536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761896.964077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048801536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761896.964084:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761896.964088:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761896.964091:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761897.219579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761897.859590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761898.243548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761898.883584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761899.267595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761899.907629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761900.291523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761900.931667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761901.315553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761901.955624:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761902.083623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761902.083631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761902.083634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761902.083643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1351 00000100:00080000:1.0:1789761902.083652:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761902.083660:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761902.083690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048801664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761902.083713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761902.083717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761902.083719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761902.083722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1351 00000100:00080000:1.0:1789761902.083725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761902.083728:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761902.083738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761902.083754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048801792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761902.083769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761902.083774:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048801664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761902.083775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761902.083780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761902.083787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1351 00000100:00080000:1.0:1789761902.083790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761902.083801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761902.083806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761902.083817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048801920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761902.083824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761902.083827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761902.083828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761902.083833:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761902.083836:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1351 00000100:00080000:1.0:1789761902.083841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761902.083844:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761902.083848:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761902.083857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048802048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761902.083864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761902.083867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761902.083868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761902.083872:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761902/real 1789761902] req@ffff9811c92c5f80 x1876700048801664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761902.083874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1351 00000100:00080000:1.0:1789761902.083878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761902.083883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761902.083904:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761902.083916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048802176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761902.083927:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048801664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761902.083948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761902.083951:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761902.083954:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048801664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761902.083974:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048801792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761902.083994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761902.084004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761902/real 1789761902] req@ffff9811c92c7480 x1876700048801792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761902.084013:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048801792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761902.084019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761902.084022:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761902.084024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048801792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761902.084028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761902.084041:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761902.084047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761902/real 1789761902] req@ffff9811c92c4a80 x1876700048801920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761902.084055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048801920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761902.084062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761902.084065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761902.084067:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048801920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761902.084077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761902.084093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761902.084102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761902/real 1789761902] req@ffff9811c92c7800 x1876700048802048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761902.084111:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048802048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761902.084119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761902.084122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761902.084124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761902.084129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761902.084144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761902.084153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761902/real 1789761902] req@ffff9811c92c7100 x1876700048802176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761902.084164:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048802176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761902.084171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761902.084174:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761902.084176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761902.339513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761902.979581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761903.363535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761904.003595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761904.387545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761905.027681:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761905.411586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761906.051641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761906.435617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761907.075628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761907.203550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761907.203562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761907.203581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761907.203589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1356 00000100:00080000:3.0:1789761907.203598:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761907.203604:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761907.203633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048802304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761907.203656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761907.203658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761907.203660:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761907.203663:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1356 00000100:00080000:3.0:1789761907.203665:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761907.203667:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761907.203674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761907.203683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048802432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761907.203691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761907.203693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761907.203695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761907.203701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1356 00000100:00100000:2.0:1789761907.203702:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048802304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761907.203703:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761907.203710:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761907.203714:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761907.203721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048802560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761907.203740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761907.203742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761907.203744:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761907.203747:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1356 00000100:00080000:3.0:1789761907.203757:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761907.203760:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761907.203764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761907.203771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048802688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761907.203776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761907.203779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761907.203781:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761907.203784:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1356 00000800:00000100:2.0:1789761907.203787:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761907.203788:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761907.203793:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761907.203796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761907.203804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048802816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761907.203828:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761907/real 1789761907] req@ffff9811e1a22300 x1876700048802304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761907.203845:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048802304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761907.203865:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761907.203869:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761907.203871:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048802304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761907.203878:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048802432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761907.203897:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761907.203906:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761907/real 1789761907] req@ffff9811e1a21180 x1876700048802432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761907.203915:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048802432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761907.203921:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761907.203924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761907.203926:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048802432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761907.203944:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761907.203956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761907.203962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761907/real 1789761907] req@ffff9811e1a22a00 x1876700048802560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761907.203977:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048802560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761907.203983:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761907.203985:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761907.203987:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761907.203991:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761907.204004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761907.204010:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761907/real 1789761907] req@ffff9811e1a21c00 x1876700048802688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761907.204018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048802688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761907.204023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761907.204025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761907.204026:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761907.204030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761907.204039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761907.204045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761907/real 1789761907] req@ffff9811e1a21880 x1876700048802816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761907.204052:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048802816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761907.204065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761907.204068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761907.204070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048802816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761907.459623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761908.099617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761908.483582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761909.123712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761909.507533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761910.147682:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761910.531584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761911.171718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761911.555631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761912.195636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761912.323549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761912.323561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761912.323589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761912.323601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1361 00000100:00080000:3.0:1789761912.323612:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761912.323620:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761912.323661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048802944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761912.323688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761912.323692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761912.323693:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761912.323696:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1361 00000100:00080000:3.0:1789761912.323699:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761912.323701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761912.323710:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761912.323721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048803072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761912.323731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761912.323734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761912.323736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761912.323741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1361 00000100:00100000:2.0:1789761912.323743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048802944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761912.323743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761912.323749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761912.323756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761912.323763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048803200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761912.323780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761912.323783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761912.323784:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761912.323787:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1361 00000100:00080000:3.0:1789761912.323790:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761912.323794:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761912.323798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761912.323808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048803328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761912.323835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761912.323838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761912.323839:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761912.323842:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1361 00000100:00080000:3.0:1789761912.323845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761912.323850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761912.323852:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761912.323859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761912.323883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048803456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761912.323916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761912/real 1789761912] req@ffff9811e1a22d80 x1876700048802944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761912.323934:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048802944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761912.323949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761912.323953:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761912.323956:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048802944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761912.323980:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048803072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761912.323998:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761912.324008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761912/real 1789761912] req@ffff9811e1a22680 x1876700048803072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761912.324016:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048803072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761912.324029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761912.324032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761912.324034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048803072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761912.324038:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761912.324054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761912.324061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761912/real 1789761912] req@ffff9811e1a21880 x1876700048803200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761912.324069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048803200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761912.324074:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761912.324076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761912.324078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761912.324082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761912.324093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761912.324099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761912/real 1789761912] req@ffff9811e1a21c00 x1876700048803328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761912.324106:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048803328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761912.324111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761912.324112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761912.324120:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761912.324124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761912.324146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761912.324153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761912/real 1789761912] req@ffff9811e1a22a00 x1876700048803456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761912.324161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048803456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761912.324167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761912.324169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761912.324171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761912.579610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761913.219600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761913.603543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761914.243643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761914.627613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761915.267643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761915.651590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761916.291640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761916.675586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761917.315631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761917.443686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761917.443722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761917.443728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761917.443742:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1367 00000100:00080000:1.0:1789761917.443759:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761917.443769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761917.443882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048803584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761917.443918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761917.443922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761917.443924:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761917.443928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1367 00000100:00080000:1.0:1789761917.443932:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761917.443936:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761917.443947:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761917.443961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048803712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761917.443972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761917.443976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761917.443979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761917.444003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048803584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761917.444006:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1367 00000100:00080000:1.0:1789761917.444014:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761917.444020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761917.444025:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761917.444036:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048803840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761917.444046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761917.444049:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761917.444051:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761917.444054:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1367 00000100:00080000:1.0:1789761917.444057:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761917.444060:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761917.444065:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761917.444075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048803968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761917.444082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761917.444085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761917.444087:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761917.444090:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1367 00000100:00080000:1.0:1789761917.444093:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761917.444097:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761917.444100:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761917.444110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048804096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761917.444124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761917.444191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761917/real 1789761917] req@ffff9811c92c6a00 x1876700048803584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761917.444219:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048803584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761917.444235:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761917.444241:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761917.444257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048803584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761917.444268:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048803712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761917.444302:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761917.444320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761917/real 1789761917] req@ffff9811c92c4700 x1876700048803712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761917.444339:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048803712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761917.444351:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761917.444356:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761917.444361:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048803712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761917.444398:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761917.444424:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761917.444439:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761917/real 1789761917] req@ffff9811c92c6300 x1876700048803840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761917.444457:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048803840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761917.444517:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761917.444523:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761917.444528:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761917.444537:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761917.444565:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761917.444592:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761917/real 1789761917] req@ffff9811c92c7100 x1876700048803968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761917.444630:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048803968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761917.444645:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761917.444651:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761917.444658:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048803968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761917.444670:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761917.444707:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761917.444745:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761917/real 1789761917] req@ffff9811c92c7800 x1876700048804096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761917.444769:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048804096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761917.444817:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761917.444841:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761917.444846:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761917.699610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761918.339618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761918.723588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761919.363597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761919.747555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761920.387707:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761920.771598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761921.411635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761921.795592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761922.435623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761922.563559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761922.563570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761922.563574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761922.563585:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1372 00000100:00080000:3.0:1789761922.563597:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761922.563605:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761922.563648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048804224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761922.563682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761922.563686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761922.563689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761922.563694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1372 00000100:00080000:3.0:1789761922.563698:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761922.563702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761922.563713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761922.563724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048804352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761922.563734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761922.563749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761922.563752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761922.563756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1372 00000100:00080000:3.0:1789761922.563760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761922.563763:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048804224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761922.563766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761922.563775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761922.563808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048804480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761922.563816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761922.563818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761922.563820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761922.563823:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1372 00000100:00080000:3.0:1789761922.563835:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761922.563838:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761922.563842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761922.563849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048804608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761922.563856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789761922.563859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761922.563862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761922.563863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761922.563867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1372 00000100:00080000:3.0:1789761922.563872:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761922.563876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761922.563881:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789761922.563894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048804736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761922.563927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761922/real 1789761922] req@ffff9811e1a21180 x1876700048804224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761922.563951:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048804224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761922.563964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761922.563969:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761922.563974:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048804224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761922.563992:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048804352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761922.564029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761922.564044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761922/real 1789761922] req@ffff9811e1a22300 x1876700048804352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761922.564069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048804352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761922.564079:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761922.564083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761922.564087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048804352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761922.564127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761922.564150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761922.564162:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761922/real 1789761922] req@ffff9811e1a23100 x1876700048804480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761922.564176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048804480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761922.564217:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761922.564222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761922.564226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761922.564233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761922.564255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761922.564267:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761922/real 1789761922] req@ffff9811e1a23800 x1876700048804608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761922.564280:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048804608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761922.564289:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761922.564293:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761922.564297:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761922.564304:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761922.564322:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761922.564332:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761922/real 1789761922] req@ffff9811e1a23480 x1876700048804736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761922.564346:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048804736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761922.564355:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761922.564358:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761922.564368:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048804736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761922.819606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761923.459686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761923.843561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761924.483646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761924.867570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761925.507580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761925.891540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761926.531772:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761926.915601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761927.555661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761927.683692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761927.683725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761927.683730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761927.683755:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1377 00000100:00080000:1.0:1789761927.683769:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761927.683778:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761927.683830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048804864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761927.683861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761927.683865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761927.683867:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761927.683871:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1377 00000100:00080000:1.0:1789761927.683873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761927.683876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761927.683887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761927.683899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048804992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761927.683911:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761927.683915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789761927.683920:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048804864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761927.683939:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761927.683946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1377 00000100:00080000:1.0:1789761927.683950:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761927.683954:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761927.683977:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761927.683993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048805120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761927.684007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761927.684011:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761927.684014:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761927.684019:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1377 00000100:00080000:1.0:1789761927.684025:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761927.684031:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761927.684055:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761927.684073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048805248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761927.684096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761927.684101:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761927.684122:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761927.684123:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048804992:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789761927.684131:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1377 00000100:00080000:1.0:1789761927.684137:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761927.684143:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761927.684160:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761927.684193:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761927.684199:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048805376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761927.684231:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761927.684264:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761927.939595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761928.579724:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761928.963599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761929.603654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761929.987590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761930.627590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761930.755777:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789761930.755823:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761930.755841:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761930.755885:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761930.755897:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761930.755910:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761930.755920:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761930.756037:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761927/real 1789761930] req@ffff9811c92c4e00 x1876700048805376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761930.756066:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048805376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761930.756080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761930.756085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761930.756090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761930.756099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761927/real 1789761930] req@ffff9811c92c7480 x1876700048804992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761930.756128:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048804992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761930.756138:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761930.756142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761930.756145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048804992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761930.756152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761927/real 1789761930] req@ffff9811c92c5f80 x1876700048805120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761930.756226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048805120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761930.756236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761930.756239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761930.756242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761930.756250:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761927/real 1789761930] req@ffff9811c92c4380 x1876700048805248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761930.756263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048805248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761930.756271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761930.756274:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761930.756278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761930.756285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761927/real 1789761930] req@ffff9811c92c4a80 x1876700048804864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761930.756308:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048804864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761930.756318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761930.756322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761930.756326:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048804864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761931.011537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761931.651661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761932.035596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761932.675859:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761932.803624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761932.803640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761932.803680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761932.803724:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1382 00000100:00080000:0.0:1789761932.803739:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761932.803751:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761932.803803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048805504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761932.803851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761932.803855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761932.803857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761932.803862:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1382 00000100:00080000:0.0:1789761932.803866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761932.803871:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761932.803883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761932.803900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048805632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761932.803907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048805504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761932.803932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761932.803936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761932.803938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761932.803944:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1382 00000100:00080000:0.0:1789761932.803963:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761932.803979:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761932.803985:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761932.804001:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048805760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761932.804011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761932.804015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761932.804020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761932.804022:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761932.804028:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1382 00000100:00080000:0.0:1789761932.804033:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761932.804037:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761932.804049:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761932.804062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048805888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761932.804072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761932.804076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761932/real 1789761932] req@ffff9811c2e28e00 x1876700048805504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761932.804077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761932.804081:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761932.804090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1382 00000100:00080000:0.0:1789761932.804104:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761932.804109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048805504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761932.804111:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761932.804119:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761932.804128:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761932.804135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761932.804137:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048806016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761932.804143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048805504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761932.804191:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048805632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761932.804220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761932.804235:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761932/real 1789761932] req@ffff9811c2e2bb80 x1876700048805632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761932.804249:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048805632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761932.804266:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761932.804270:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761932.804274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048805632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761932.804281:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761932.804307:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761932.804319:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761932/real 1789761932] req@ffff9811c2e29180 x1876700048805760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761932.804330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048805760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761932.804338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761932.804342:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761932.804345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761932.804351:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761932.804373:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761932.804385:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761932/real 1789761932] req@ffff9811c2e29880 x1876700048805888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761932.804398:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048805888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761932.804406:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761932.804416:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761932.804430:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048805888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761932.804438:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761932.804491:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761932.804505:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761932/real 1789761932] req@ffff9811c2e2ad80 x1876700048806016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761932.804518:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048806016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761932.804526:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761932.804529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761932.804533:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761933.059595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761933.699610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761934.083542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761934.723603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761935.107686:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761935.747666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761936.131580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761936.771651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761937.155532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761937.795591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761937.923557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761937.923566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761937.923570:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761937.923578:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1387 00000100:00080000:3.0:1789761937.923587:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761937.923594:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789761937.923639:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048806144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761937.923663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761937.923667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761937.923669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761937.923672:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1387 00000100:00080000:3.0:1789761937.923676:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761937.923678:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761937.923692:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761937.923705:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048806144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761937.923706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048806272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761937.923725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761937.923728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761937.923729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761937.923733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1387 00000100:00080000:3.0:1789761937.923737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761937.923740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761937.923745:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761937.923753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048806400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761937.923760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761937.923763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761937.923765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761937.923767:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1387 00000100:00080000:3.0:1789761937.923770:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761937.923774:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761937.923777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761937.923784:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761937.923787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048806528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761937.923824:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761937/real 1789761937] req@ffff9811e1a20000 x1876700048806144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761937.923842:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048806144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761937.923852:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761937.923854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761937.923858:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761937.923862:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048806144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789761937.923869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789761937.923879:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761937.923884:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1387 00000100:00100000:0.0:1789761937.923888:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048806272:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789761937.923889:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761937.923897:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789761937.923902:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761937.923912:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789761937.923916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048806656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761937.923925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761937/real 1789761937] req@ffff9811e1a20700 x1876700048806272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761937.923935:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048806272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761937.923953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761937.923955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761937.923958:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048806272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761937.923962:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761937.923977:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761937.923984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761937/real 1789761937] req@ffff9811e1a20380 x1876700048806400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761937.923992:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048806400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761937.923998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761937.924000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761937.924002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761937.924007:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761937.924021:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761937.924028:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761937/real 1789761937] req@ffff9811e1a20e00 x1876700048806528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761937.924037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048806528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761937.924042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761937.924044:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761937.924053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761937.924069:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761937.924086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761937.924093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761937/real 1789761937] req@ffff9811e1a21500 x1876700048806656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761937.924101:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048806656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761937.924108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761937.924110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761937.924113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048806656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761938.179538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761938.819612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761939.203529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761939.843629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761940.227686:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761940.867574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761941.251553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761941.891645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761942.275549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761942.915551:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761943.043561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761943.043569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761943.043573:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761943.043582:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1392 00000100:00080000:0.0:1789761943.043603:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761943.043610:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761943.043655:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048806784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761943.043676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761943.043679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761943.043681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761943.043684:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1392 00000100:00080000:0.0:1789761943.043699:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761943.043703:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761943.043712:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761943.043722:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048806784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761943.043724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048806912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761943.043737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761943.043740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761943.043742:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761943.043745:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1392 00000100:00080000:0.0:1789761943.043747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761943.043749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761943.043753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761943.043760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048807040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761943.043767:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761943.043770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761943.043771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761943.043773:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1392 00000100:00080000:0.0:1789761943.043776:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761943.043778:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761943.043782:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761943.043792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048807168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761943.043792:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761943.043811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761943.043813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761943.043815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761943.043818:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1392 00000100:00080000:0.0:1789761943.043821:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761943.043824:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761943.043829:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761943.043836:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761943/real 1789761943] req@ffff9811c2e28700 x1876700048806784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761943.043840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048807296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761943.043876:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048806784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761943.043889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761943.043892:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761943.043895:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048806784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761943.043913:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048806912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761943.043931:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761943.043939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761943/real 1789761943] req@ffff9811c2e2a680 x1876700048806912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761943.043947:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048806912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761943.043953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761943.043955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761943.043957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048806912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761943.043961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761943.043970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761943.043976:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761943/real 1789761943] req@ffff9811c2e29c00 x1876700048807040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761943.043983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048807040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761943.043997:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761943.043999:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761943.044001:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761943.044005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761943.044018:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761943.044024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761943/real 1789761943] req@ffff9811c2e2ad80 x1876700048807168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761943.044031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048807168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761943.044036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761943.044038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761943.044040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761943.044043:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761943.044053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761943.044058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761943/real 1789761943] req@ffff9811c2e29880 x1876700048807296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761943.044065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048807296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761943.044070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761943.044072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761943.044079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761943.299532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761943.939668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761944.323604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761944.963610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761945.347600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789761945.987692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761946.371634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761947.011602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761947.395626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761948.035701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761948.163571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761948.163587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761948.163592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761948.163603:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1397 00000100:00080000:0.0:1789761948.163632:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761948.163640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761948.163684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048807424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761948.163708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761948.163711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761948.163712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761948.163715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1397 00000100:00080000:0.0:1789761948.163736:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761948.163739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761948.163749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761948.163778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048807552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761948.163794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761948.163799:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048807424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761948.163801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761948.163808:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761948.163829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1397 00000100:00080000:0.0:1789761948.163834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761948.163840:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761948.163846:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761948.163863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048807680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761948.163872:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761948.163887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761948.163892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761948.163894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761948.163902:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1397 00000100:00080000:0.0:1789761948.163909:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761948.163915:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761948.163918:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761948/real 1789761948] req@ffff9811c2e29180 x1876700048807424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761948.163936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048807424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761948.163940:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789761948.163957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761948.163966:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048807808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761948.163967:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761948.163982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048807424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761948.164005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761948.164008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761948.164010:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761948.164012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048807552:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789761948.164028:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1397 00000100:00080000:0.0:1789761948.164034:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761948.164039:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761948.164044:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789761948.164052:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761948.164066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761948/real 1789761948] req@ffff9811c2e2bb80 x1876700048807552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761948.164071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048807936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761948.164093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048807552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761948.164113:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761948.164129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761948.164134:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048807552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761948.164141:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761948.164175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761948.164188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761948/real 1789761948] req@ffff9811c2e28e00 x1876700048807680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761948.164202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048807680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761948.164212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761948.164215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761948.164219:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761948.164226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761948.164246:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761948.164255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761948/real 1789761948] req@ffff9811c2e2b100 x1876700048807808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761948.164263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048807808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761948.164269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761948.164271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761948.164273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761948.164286:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761948.164306:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761948.164312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761948/real 1789761948] req@ffff9811c2e2aa00 x1876700048807936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761948.164320:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048807936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761948.164326:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761948.164328:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761948.164338:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048807936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761948.419570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761949.059672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761949.443580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761950.083672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761950.467581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761951.107612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761951.491585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761952.131631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761952.515583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761953.155625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761953.283544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761953.283554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761953.283558:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761953.283568:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1402 00000100:00080000:1.0:1789761953.283583:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761953.283593:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761953.283638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048808064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761953.283663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761953.283668:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761953.283671:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761953.283675:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1402 00000100:00080000:1.0:1789761953.283680:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761953.283684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761953.283693:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761953.283694:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048808064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761953.283707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048808192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761953.283718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761953.283722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761953.283724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761953.283737:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1402 00000100:00080000:1.0:1789761953.283747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761953.283752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789761953.283753:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789761953.283757:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761953.283768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048808320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761953.283779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761953.283782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761953.283784:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761953.283788:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1402 00000100:00100000:0.0:1789761953.283790:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761953/real 1789761953] req@ffff9811c92c4000 x1876700048808064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761953.283791:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761953.283796:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761953.283801:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761953.283810:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048808064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761953.283810:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048808448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761953.283820:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761953.283822:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761953.283824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761953.283826:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789761953.283826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761953.283830:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1402 00000100:00100000:0.0:1789761953.283831:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048808064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761953.283833:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761953.283836:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761953.283840:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761953.283848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048808576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761953.283854:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048808192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761953.283877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761953.283898:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761953/real 1789761953] req@ffff9811c92c6680 x1876700048808192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761953.283912:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048808192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761953.283922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761953.283924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761953.283926:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048808192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761953.283931:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761953.283947:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761953.283955:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761953/real 1789761953] req@ffff9811c92c5c00 x1876700048808320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761953.283963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048808320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761953.283969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761953.283971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761953.283973:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761953.283976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761953.283986:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761953.283995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761953/real 1789761953] req@ffff9811c92c5880 x1876700048808448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761953.284015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048808448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761953.284032:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761953.284035:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761953.284039:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761953.284055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761953.284082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761953.284096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761953/real 1789761953] req@ffff9811c92c5500 x1876700048808576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761953.284109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048808576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761953.284117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761953.284120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761953.284123:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761953.539594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761954.179693:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761954.563592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761955.203632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761955.587599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761956.227645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761956.611539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761957.251645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761957.635600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761958.275627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761958.403558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761958.403569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761958.403573:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761958.403584:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1407 00000100:00080000:1.0:1789761958.403595:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761958.403602:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761958.403651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048808704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761958.403679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761958.403682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761958.403684:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761958.403687:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1407 00000100:00080000:1.0:1789761958.403689:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761958.403691:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761958.403701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761958.403712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048808832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761958.403721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761958.403724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761958.403726:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761958.403729:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048808704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761958.403731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1407 00000100:00080000:1.0:1789761958.403738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761958.403741:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761958.403762:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761958.403773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048808960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761958.403788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761958.403791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761958.403795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761958.403798:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1407 00000100:00080000:1.0:1789761958.403802:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761958.403805:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761958.403809:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789761958.403815:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761958.403821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048809088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761958.403828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761958.403831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761958.403832:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761958.403835:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1407 00000100:00080000:1.0:1789761958.403838:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761958.403840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761958.403844:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761958.403850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048809216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761958.403878:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761958/real 1789761958] req@ffff9811c92c7b80 x1876700048808704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761958.403898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048808704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761958.403916:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761958.403919:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761958.403922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048808704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761958.403944:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048808832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761958.403963:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761958.403972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761958/real 1789761958] req@ffff9811c92c5180 x1876700048808832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761958.403989:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048808832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761958.403995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761958.403997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761958.403999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048808832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761958.404003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789761958.404015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761958.404022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761958/real 1789761958] req@ffff9811c92c6d80 x1876700048808960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761958.404029:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048808960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761958.404034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761958.404036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761958.404038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048808960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789761958.404047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761958.404061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761958.404068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761958/real 1789761958] req@ffff9811c92c5500 x1876700048809088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761958.404075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048809088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761958.404084:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761958.404086:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761958.404088:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789761958.404092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789761958.404103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789761958.404109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761958/real 1789761958] req@ffff9811c92c5880 x1876700048809216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761958.404121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048809216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761958.404128:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761958.404132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789761958.404135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761958.659562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761959.299622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761959.683574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761960.323631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761960.707534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761961.347623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789761961.731557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761962.371618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761962.755573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761963.395646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761963.523589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761963.523623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761963.523628:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761963.523639:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1413 00000100:00080000:0.0:1789761963.523650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761963.523658:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761963.523699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048809344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761963.523729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761963.523732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761963.523734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761963.523738:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1413 00000100:00080000:0.0:1789761963.523742:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761963.523747:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761963.523757:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761963.523773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048809472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761963.523788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761963.523792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789761963.523792:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048809344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761963.523796:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761963.523804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1413 00000100:00080000:0.0:1789761963.523807:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761963.523810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761963.523815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761963.523826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048809600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761963.523839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761963.523842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761963.523843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761963.523846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1413 00000100:00080000:0.0:1789761963.523850:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761963.523852:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761963.523857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761963.523869:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048809728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761963.523869:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761963.523878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761963.523881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761963.523883:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761963.523886:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1413 00000100:00080000:0.0:1789761963.523888:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761963.523891:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761963.523894:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761963.523902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048809856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761963.523923:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761963/real 1789761963] req@ffff9811c2e28380 x1876700048809344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761963.523940:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048809344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761963.523949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761963.523952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761963.523956:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048809344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761963.523977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048809472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761963.523993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761963.524001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761963/real 1789761963] req@ffff9811c2e28a80 x1876700048809472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761963.524009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048809472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761963.524015:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761963.524017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761963.524020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048809472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761963.524023:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761963.524033:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761963.524048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761963/real 1789761963] req@ffff9811c2e29f80 x1876700048809600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761963.524056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048809600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761963.524061:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761963.524062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761963.524064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761963.524068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761963.524081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761963.524086:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761963/real 1789761963] req@ffff9811c2e2b800 x1876700048809728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761963.524093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048809728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761963.524098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761963.524099:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761963.524101:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761963.524104:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761963.524115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761963.524120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761963/real 1789761963] req@ffff9811c2e2b480 x1876700048809856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761963.524127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048809856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761963.524138:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761963.524140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761963.524142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048809856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761963.779583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761964.419578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761964.803547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761965.443614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761965.827546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761966.467627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761966.851610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761967.491589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761967.875593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761968.515578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761968.643628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761968.643644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761968.643650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761968.643664:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1418 00000100:00080000:0.0:1789761968.643723:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761968.643734:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761968.643820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048809984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761968.643896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761968.643903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761968.643907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761968.643914:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1418 00000100:00080000:0.0:1789761968.643919:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761968.643926:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761968.643942:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761968.643990:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048810112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761968.644004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761968.644008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761968.644010:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761968.644017:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1418 00000100:00100000:2.0:1789761968.644021:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048809984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789761968.644022:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761968.644030:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761968.644037:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761968.644052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048810240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761968.644064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761968.644075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761968.644078:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761968.644083:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1418 00000100:00080000:0.0:1789761968.644088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761968.644093:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789761968.644099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789761968.644100:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761968.644120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048810368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761968.644131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761968.644135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761968.644138:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761968.644144:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1418 00000100:00080000:0.0:1789761968.644148:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761968.644151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761968/real 1789761968] req@ffff9811c2e2a300 x1876700048809984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761968.644156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761968.644167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761968.644177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048809984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761968.644183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048810496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761968.644190:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761968.644198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761968.644203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048809984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761968.644210:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048810112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761968.644228:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761968.644237:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761968/real 1789761968] req@ffff9811c2e28000 x1876700048810112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761968.644245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048810112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761968.644262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761968.644264:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761968.644267:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048810112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761968.644289:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761968.644304:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761968.644310:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761968/real 1789761968] req@ffff9811c2e29500 x1876700048810240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761968.644318:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048810240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761968.644324:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761968.644326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761968.644328:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761968.644331:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761968.644342:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761968.644349:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761968/real 1789761968] req@ffff9811c2e2b480 x1876700048810368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761968.644361:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048810368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761968.644369:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761968.644372:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761968.644382:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761968.644389:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761968.644427:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761968.644446:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761968/real 1789761968] req@ffff9811c2e2b800 x1876700048810496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761968.644454:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048810496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761968.644460:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761968.644485:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761968.644488:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761968.899601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761969.539594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761969.923568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761970.563638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761970.947541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789761971.587587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761971.971560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761972.611579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761972.995578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761973.635732:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761973.763558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761973.763566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761973.763570:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761973.763579:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1423 00000100:00080000:0.0:1789761973.763593:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761973.763603:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789761973.763656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048810624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761973.763684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761973.763687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761973.763689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761973.763693:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1423 00000100:00080000:0.0:1789761973.763699:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761973.763705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761973.763714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761973.763731:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048810624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789761973.763742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048810752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761973.763764:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761973.763768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761973.763770:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761973.763773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1423 00000100:00080000:0.0:1789761973.763776:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761973.763779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761973.763788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761973.763797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048810880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761973.763806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761973.763809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761973.763811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789761973.763816:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789761973.763821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1423 00000100:00080000:0.0:1789761973.763825:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761973.763828:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761973.763839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789761973.763851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048811008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761973.763860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761973.763863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789761973.763864:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761973.763867:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1423 00000100:00080000:0.0:1789761973.763871:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761973.763877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789761973.763883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761973.763888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761973/real 1789761973] req@ffff9811c2e29f80 x1876700048810624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761973.763900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048811136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761973.763909:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048810624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761973.763928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761973.763944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761973.763952:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048810624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761973.763989:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048810752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761973.764016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761973.764031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761973/real 1789761973] req@ffff9811c2e28a80 x1876700048810752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761973.764042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048810752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761973.764050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761973.764054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761973.764057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048810752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761973.764062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761973.764081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761973.764091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761973/real 1789761973] req@ffff9811c2e28380 x1876700048810880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761973.764103:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048810880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761973.764123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761973.764127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761973.764131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048810880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761973.764138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761973.764164:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761973.764176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761973/real 1789761973] req@ffff9811c2e2aa00 x1876700048811008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761973.764189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048811008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761973.764198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761973.764203:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761973.764205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761973.764212:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761973.764236:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761973.764247:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761973/real 1789761973] req@ffff9811c2e2b100 x1876700048811136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761973.764265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048811136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761973.764271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761973.764274:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761973.764284:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761974.019587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761974.659561:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761975.043562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761975.683608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761976.067554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761976.707562:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761977.091562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789761977.731670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761978.115538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761978.755607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761978.883597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761978.883615:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761978.883619:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761978.883630:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1428 00000100:00080000:1.0:1789761978.883643:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761978.883650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761978.883694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048811264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761978.883722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761978.883725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761978.883727:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761978.883730:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1428 00000100:00080000:1.0:1789761978.883732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761978.883734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761978.883744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761978.883758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048811392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761978.883770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761978.883775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761978.883777:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761978.883784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1428 00000100:00100000:2.0:1789761978.883785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048811264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789761978.883789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761978.883816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761978.883823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761978.883838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048811520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761978.883851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761978.883855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761978.883866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761978.883870:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761978.883877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1428 00000100:00080000:1.0:1789761978.883884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761978.883888:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761978.883893:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761978.883903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048811648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761978.883910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761978.883913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761978.883914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761978.883917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1428 00000100:00080000:1.0:1789761978.883920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761978.883923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761978.883927:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761978.883929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761978/real 1789761978] req@ffff9811c92c5c00 x1876700048811264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761978.883940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048811776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761978.883958:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048811264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761978.883971:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761978.883976:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761978.883991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048811264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761978.884000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048811392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761978.884027:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761978.884042:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761978/real 1789761978] req@ffff9811c92c6680 x1876700048811392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761978.884056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048811392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761978.884066:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761978.884069:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761978.884073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048811392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761978.884102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761978.884135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761978.884147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761978/real 1789761978] req@ffff9811c92c4000 x1876700048811520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761978.884160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048811520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761978.884169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761978.884173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761978.884177:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761978.884184:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761978.884202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761978.884222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761978/real 1789761978] req@ffff9811c92c4a80 x1876700048811648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761978.884235:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048811648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761978.884244:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761978.884248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761978.884252:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761978.884258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761978.884279:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761978.884290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761978/real 1789761978] req@ffff9811c92c4380 x1876700048811776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761978.884302:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048811776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761978.884311:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761978.884332:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761978.884336:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048811776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761979.139555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761979.779579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761980.163550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761980.803550:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761981.187560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761981.827570:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761982.211584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761982.851584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761983.235582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761983.875584:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761984.003593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761984.003605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761984.003627:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761984.003636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1433 00000100:00080000:2.0:1789761984.003646:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761984.003652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761984.003682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048811904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761984.003714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761984.003717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761984.003718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761984.003722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1433 00000100:00080000:2.0:1789761984.003724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761984.003726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761984.003733:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761984.003742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048812032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761984.003751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761984.003754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761984.003755:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761984.003759:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1433 00000100:00080000:2.0:1789761984.003763:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789761984.003764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048811904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789761984.003767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761984.003771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761984.003782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048812160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761984.003802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761984.003804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761984.003806:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761984.003810:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1433 00000100:00080000:2.0:1789761984.003813:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761984.003816:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761984.003820:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789761984.003829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761984.003832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048812288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761984.003840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761984.003843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761984.003844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761984.003847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1433 00000100:00080000:2.0:1789761984.003851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761984.003855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761984.003860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761984.003874:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761984/real 1789761984] req@ffff9811d8638e00 x1876700048811904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761984.003875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048812416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761984.003905:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048811904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761984.003919:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761984.003924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761984.003930:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048811904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761984.003940:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048812032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761984.003961:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761984.003972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761984/real 1789761984] req@ffff9811d8638700 x1876700048812032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761984.003981:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048812032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761984.003987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761984.003990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761984.003992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048812032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761984.004010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761984.004022:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761984.004029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761984/real 1789761984] req@ffff9811d863a300 x1876700048812160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761984.004047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048812160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761984.004053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761984.004055:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761984.004057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761984.004060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761984.004073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761984.004080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761984/real 1789761984] req@ffff9811d8638380 x1876700048812288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761984.004087:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048812288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761984.004092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761984.004094:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761984.004096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761984.004099:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761984.004108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761984.004113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761984/real 1789761984] req@ffff9811d863b480 x1876700048812416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761984.004120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048812416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761984.004133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761984.004136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761984.004138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761984.259619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761984.899640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761985.283570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761985.923592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761986.307590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761986.947599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761987.331537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761987.971571:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761988.355587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761988.995568:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761989.123585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761989.123594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761989.123597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761989.123605:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1438 00000100:00080000:1.0:1789761989.123615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761989.123621:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761989.123653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048812544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761989.123678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761989.123681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761989.123682:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761989.123685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1438 00000100:00080000:1.0:1789761989.123688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761989.123690:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761989.123699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761989.123711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048812544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761989.123718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048812672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761989.123731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761989.123734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761989.123747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761989.123751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1438 00000100:00080000:1.0:1789761989.123754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761989.123758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761989.123762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761989.123772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048812800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761989.123779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761989.123784:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789761989.123785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761989.123788:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761989.123792:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1438 00000100:00080000:1.0:1789761989.123795:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761989.123797:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761989.123801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761989.123808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048812928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761989.123814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761989.123817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761989.123818:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761989.123821:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1438 00000100:00080000:1.0:1789761989.123824:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761989.123829:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761989.123830:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761989/real 1789761989] req@ffff9811c92c5f80 x1876700048812544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761989.123832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789761989.123848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048813056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761989.123860:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048812544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761989.123880:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761989.123883:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761989.123886:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048812544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761989.123907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048812672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761989.123926:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761989.123935:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761989/real 1789761989] req@ffff9811c92c7480 x1876700048812672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761989.123944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048812672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761989.123949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761989.123952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761989.123954:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048812672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761989.123957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789761989.123968:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761989.123974:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761989/real 1789761989] req@ffff9811c92c4e00 x1876700048812800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761989.123994:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048812800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761989.124000:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761989.124003:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761989.124007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761989.124013:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761989.124035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761989.124046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761989/real 1789761989] req@ffff9811c92c7800 x1876700048812928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761989.124059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048812928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761989.124065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761989.124067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761989.124069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048812928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761989.124073:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789761989.124087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789761989.124098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761989/real 1789761989] req@ffff9811c92c7100 x1876700048813056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761989.124105:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048813056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761989.124120:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761989.124122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761989.124125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761989.379548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761990.019595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761990.403558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761991.043575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761991.427560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761992.067644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761992.451603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761993.091612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761993.475543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761994.115587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761994.243582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761994.243590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761994.243593:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761994.243602:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1443 00000100:00080000:1.0:1789761994.243612:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761994.243618:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761994.243651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048813184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761994.243674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761994.243678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761994.243680:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761994.243692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1443 00000100:00080000:1.0:1789761994.243697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761994.243701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761994.243711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048813184:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789761994.243712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761994.243732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048813312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761994.243747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761994.243750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761994.243752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761994.243755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1443 00000100:00080000:1.0:1789761994.243759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761994.243777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761994.243782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761994.243791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048813440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761994.243798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761994.243801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761994.243803:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761994.243806:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1443 00000100:00080000:1.0:1789761994.243809:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761994.243812:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761994.243815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761994.243826:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048813312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761994.243827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048813568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761994.243841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761994.243843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789761994.243844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761994.243847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1443 00000100:00080000:1.0:1789761994.243851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761994.243854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789761994.243859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761994.243864:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761994.243870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048813696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761994.243896:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761994.243924:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761994.499596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761995.139694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761995.523547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761996.163561:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761996.547555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761997.187560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761997.315744:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789761997.315804:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761997.315825:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761997.315880:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761997.315890:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761997.315898:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789761997.315906:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789761997.315981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761994/real 1789761997] req@ffff9811c92c7100 x1876700048813568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761997.316011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048813568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761997.316023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761997.316046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761997.316051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761997.316061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761994/real 1789761997] req@ffff9811c92c7800 x1876700048813696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761997.316077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048813696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761997.316087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761997.316091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761997.316094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789761997.316101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761994/real 1789761997] req@ffff9811c92c4700 x1876700048813312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761997.316114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048813312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761997.316123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761997.316127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761997.316130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048813312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761997.316138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761994/real 1789761997] req@ffff9811c92c6a00 x1876700048813440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761997.316151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048813440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761997.316158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761997.316170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761997.316174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048813440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789761997.316199:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761994/real 1789761997] req@ffff9811c92c6300 x1876700048813184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761997.316227:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048813184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761997.316236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761997.316240:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789761997.316243:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048813184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789761997.571524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761998.211584:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789761998.595537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789761999.235616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761999.363594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761999.363603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761999.363607:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761999.363615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1448 00000100:00080000:2.0:1789761999.363625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761999.363632:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789761999.363662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048813824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761999.363692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761999.363697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761999.363699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761999.363703:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1448 00000100:00080000:2.0:1789761999.363707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761999.363726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761999.363745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761999.363746:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048813824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789761999.363758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048813952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761999.363770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761999.363773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761999.363775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761999.363778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1448 00000100:00080000:2.0:1789761999.363782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761999.363786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761999.363791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761999.363800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048814080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761999.363809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761999.363811:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789761999.363813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761999.363814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761999.363817:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1448 00000100:00080000:2.0:1789761999.363820:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761999.363823:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789761999.363828:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761999.363836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048814208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761999.363844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761999.363851:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761999/real 1789761999] req@ffff9811d863b800 x1876700048813824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761999.363853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789761999.363855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761999.363859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1448 00000100:00080000:2.0:1789761999.363863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761999.363867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789761999.363870:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048813824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761999.363872:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789761999.363881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048814336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761999.363898:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761999.363903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761999.363907:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048813824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789761999.363932:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048813952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761999.363958:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761999.363978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761999/real 1789761999] req@ffff9811d8638000 x1876700048813952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761999.363991:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048813952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761999.364000:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761999.364004:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761999.364007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048813952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761999.364014:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789761999.364036:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761999.364048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761999/real 1789761999] req@ffff9811d863aa00 x1876700048814080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761999.364075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048814080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761999.364084:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761999.364087:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761999.364090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789761999.364096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761999.364119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761999.364131:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761999/real 1789761999] req@ffff9811d8638a80 x1876700048814208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761999.364142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048814208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761999.364150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761999.364153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761999.364156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789761999.364178:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789761999.364201:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789761999.364212:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761999/real 1789761999] req@ffff9811d863a680 x1876700048814336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761999.364223:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048814336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761999.364241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761999.364245:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789761999.364248:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789761999.619571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762000.259636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762000.643534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762001.283604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762001.667510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762002.307617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762002.691518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762003.331590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762003.715520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762004.355627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762004.483578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762004.483589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762004.483594:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762004.483605:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1454 00000100:00080000:1.0:1789762004.483617:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762004.483625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762004.483669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048814464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762004.483699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762004.483703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762004.483705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762004.483710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1454 00000100:00080000:1.0:1789762004.483714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762004.483719:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762004.483731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762004.483743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048814464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762004.483745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048814592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762004.483761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762004.483763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762004.483783:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762004.483787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1454 00000100:00080000:1.0:1789762004.483791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762004.483794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762004.483799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762004.483808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048814720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762004.483816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762004.483818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762004.483819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762004.483822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1454 00000100:00080000:1.0:1789762004.483824:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762004.483826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762004.483830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762004.483832:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762004.483841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048814848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762004.483849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762004.483852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762004.483855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762004.483859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1454 00000100:00080000:1.0:1789762004.483863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762004.483867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762004.483872:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762004.483892:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762004/real 1789762004] req@ffff9811c92c4e00 x1876700048814464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762004.483897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048814976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762004.483934:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048814464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762004.483948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762004.483952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762004.483957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048814464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762004.483984:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048814592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762004.484006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762004.484026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762004/real 1789762004] req@ffff9811c92c7480 x1876700048814592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762004.484040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048814592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762004.484050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762004.484053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762004.484057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048814592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762004.484063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762004.484081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762004.484090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762004/real 1789762004] req@ffff9811c92c5f80 x1876700048814720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762004.484103:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048814720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762004.484112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762004.484115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762004.484119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762004.484140:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762004.484163:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762004.484176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762004/real 1789762004] req@ffff9811c92c4380 x1876700048814848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762004.484188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048814848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762004.484203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762004.484207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762004.484211:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762004.484217:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762004.484242:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762004.484255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762004/real 1789762004] req@ffff9811c92c4a80 x1876700048814976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762004.484268:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048814976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762004.484278:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762004.484281:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762004.484285:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048814976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762004.739505:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762005.379626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762005.763508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762006.403576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762006.787534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762007.427586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762007.811543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762008.451611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762008.835539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762009.475628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762009.603621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762009.603635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762009.603639:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762009.603651:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1459 00000100:00080000:1.0:1789762009.603664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762009.603672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762009.603719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048815104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762009.603744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762009.603747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762009.603749:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762009.603752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1459 00000100:00080000:1.0:1789762009.603754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762009.603757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762009.603766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762009.603776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048815232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762009.603785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762009.603788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762009.603789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762009.603792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1459 00000100:00080000:1.0:1789762009.603794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762009.603804:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048815104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762009.603823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762009.603828:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762009.603838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048815360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762009.603847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762009.603849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762009.603851:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762009.603853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1459 00000100:00080000:1.0:1789762009.603857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762009.603860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762009.603867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762009.603875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048815488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762009.603881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762009.603883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762009.603887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762009.603889:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762009.603892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1459 00000100:00080000:1.0:1789762009.603896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762009.603898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762009.603901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762009.603909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048815616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762009.603946:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762009/real 1789762009] req@ffff9811c92c4000 x1876700048815104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762009.603969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048815104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762009.603995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762009.604001:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762009.604007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048815104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762009.604018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048815232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762009.604050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762009.604064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762009/real 1789762009] req@ffff9811c92c6680 x1876700048815232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762009.604078:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048815232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762009.604105:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762009.604109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762009.604113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048815232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762009.604154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762009.604180:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762009.604191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762009/real 1789762009] req@ffff9811c92c5c00 x1876700048815360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762009.604208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048815360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762009.604217:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762009.604222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762009.604226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762009.604246:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762009.604269:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762009.604283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762009/real 1789762009] req@ffff9811c92c5880 x1876700048815488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762009.604300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048815488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762009.604310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762009.604316:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762009.604321:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762009.604329:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762009.604347:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762009.604358:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762009/real 1789762009] req@ffff9811c92c5500 x1876700048815616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762009.604370:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048815616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762009.604379:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762009.604383:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762009.604386:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048815616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762009.859590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762010.499594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762010.883571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762011.523586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762011.907588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762012.547635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762012.931564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762013.571552:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762013.955573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762014.595693:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762014.723552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762014.723578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762014.723582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762014.723592:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1464 00000100:00080000:1.0:1789762014.723602:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762014.723610:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762014.723646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048815744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762014.723677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762014.723682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762014.723684:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762014.723689:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1464 00000100:00080000:1.0:1789762014.723692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762014.723697:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762014.723707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762014.723720:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048815744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762014.723737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048815872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762014.723751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762014.723756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762014.723758:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762014.723763:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1464 00000100:00080000:1.0:1789762014.723768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762014.723772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762014.723777:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762014.723787:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762014.723791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048816000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762014.723812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762014.723816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762014.723819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762014.723825:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1464 00000100:00100000:2.0:1789762014.723826:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762014/real 1789762014] req@ffff9811c92c6d80 x1876700048815744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762014.723832:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762014.723841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762014.723848:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762014.723848:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048815744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762014.723862:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762014.723868:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762014.723869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048816128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762014.723872:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048815744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762014.723889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762014.723893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762014.723895:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762014.723899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048815872:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762014.723904:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1464 00000100:00080000:1.0:1789762014.723910:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762014.723915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762014.723921:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762014.723921:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762014.723934:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762014/real 1789762014] req@ffff9811c92c5180 x1876700048815872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762014.723939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048816256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762014.723951:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048815872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762014.723972:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762014.723975:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762014.723980:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048815872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762014.723984:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762014.723997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762014.724004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762014/real 1789762014] req@ffff9811c92c7b80 x1876700048816000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762014.724022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048816000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762014.724028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762014.724030:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762014.724032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762014.724035:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762014.724068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762014.724078:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762014/real 1789762014] req@ffff9811c92c5500 x1876700048816128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762014.724085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048816128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762014.724091:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762014.724093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762014.724095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762014.724107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762014.724119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762014.724126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762014/real 1789762014] req@ffff9811c92c5880 x1876700048816256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762014.724133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048816256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762014.724147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762014.724149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762014.724151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762014.979592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762015.619639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762016.003578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762016.643622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762017.027599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762017.667629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762018.051565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762018.691626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762019.075614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762019.715601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762019.843553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762019.843577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762019.843582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762019.843596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1469 00000100:00080000:1.0:1789762019.843628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762019.843648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762019.843695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048816384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762019.843726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762019.843730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762019.843733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762019.843737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1469 00000100:00080000:1.0:1789762019.843741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762019.843746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762019.843758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762019.843772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048816512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762019.843787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762019.843796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762019.843798:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048816384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762019.843800:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762019.843818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1469 00000100:00080000:1.0:1789762019.843832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762019.843843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762019.843851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762019.843871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048816640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762019.843873:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762019.843890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762019.843901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762019.843903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762019.843911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1469 00000100:00080000:1.0:1789762019.843917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762019.843921:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762019/real 1789762019] req@ffff9811c92c5c00 x1876700048816384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762019.843928:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762019.843948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762019.843956:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048816384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762019.843968:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762019.843970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048816768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762019.843976:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762019.843989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048816384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762019.844029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762019.844034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762019.844036:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762019.844043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1469 00000100:00080000:1.0:1789762019.844047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762019.844053:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048816512:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762019.844058:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762019.844074:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762019.844090:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048816896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762019.844094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762019.844128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762019/real 1789762019] req@ffff9811c92c6680 x1876700048816512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762019.844144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048816512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762019.844169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762019.844172:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762019.844174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048816512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762019.844180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762019.844197:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762019.844205:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762019/real 1789762019] req@ffff9811c92c4000 x1876700048816640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762019.844213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048816640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762019.844219:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762019.844221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762019.844223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762019.844227:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762019.844238:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762019.844244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762019/real 1789762019] req@ffff9811c92c4a80 x1876700048816768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762019.844251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048816768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762019.844256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762019.844258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762019.844265:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762019.844279:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762019.844306:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762019.844320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762019/real 1789762019] req@ffff9811c92c4380 x1876700048816896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762019.844334:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048816896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762019.844343:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762019.844347:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762019.844350:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048816896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762020.099557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762020.739663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762021.123602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762021.763591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762022.147551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762022.787676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762023.171560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762023.811551:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762024.195559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762024.835590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762024.963610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762024.963624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762024.963630:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762024.963654:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1474 00000100:00080000:3.0:1789762024.963668:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762024.963677:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762024.963724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048817024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762024.963782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762024.963786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762024.963788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762024.963793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1474 00000100:00080000:3.0:1789762024.963797:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762024.963802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762024.963812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762024.963825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048817152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762024.963838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762024.963842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762024.963844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762024.963848:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1474 00000100:00080000:3.0:1789762024.963852:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762024.963856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762024.963862:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762024.963875:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048817024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762024.963875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048817280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762024.963901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762024.963905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762024.963909:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762024.963915:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1474 00000100:00080000:3.0:1789762024.963947:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762024.963953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789762024.963956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789762024.963990:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762024.964010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048817408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762024.964021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762024.964025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762024.964027:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762024.964032:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1474 00000100:00080000:3.0:1789762024.964036:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762024.964043:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762024/real 1789762024] req@ffff9811e1a21f80 x1876700048817024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762024.964046:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762024.964055:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762024.964082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048817536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762024.964084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048817024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762024.964144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762024.964153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762024.964156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048817024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762024.964163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048817152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762024.964198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762024.964208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762024/real 1789762024] req@ffff9811e1a20a80 x1876700048817152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762024.964217:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048817152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762024.964223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762024.964226:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762024.964228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048817152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762024.964249:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762024.964260:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762024.964267:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762024/real 1789762024] req@ffff9811e1a21500 x1876700048817280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762024.964275:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048817280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762024.964281:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762024.964283:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762024.964285:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762024.964289:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762024.964299:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762024.964305:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762024/real 1789762024] req@ffff9811e1a20e00 x1876700048817408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762024.964324:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048817408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762024.964330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762024.964332:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762024.964334:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762024.964337:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762024.964351:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762024.964357:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762024/real 1789762024] req@ffff9811e1a20380 x1876700048817536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762024.964373:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048817536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762024.964378:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762024.964380:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762024.964382:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762025.219629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762025.859589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762026.243558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762026.883621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762027.267555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762027.907624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762028.291651:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762028.931621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762029.315566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762029.955588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762030.083576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762030.083590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762030.083596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762030.083610:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1479 00000100:00080000:3.0:1789762030.083625:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762030.083635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762030.083690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048817664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762030.083737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762030.083741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762030.083742:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762030.083746:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1479 00000100:00080000:3.0:1789762030.083749:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762030.083754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762030.083765:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762030.083787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048817792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762030.083788:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048817664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762030.083804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762030.083807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762030.083809:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762030.083811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1479 00000100:00080000:3.0:1789762030.083814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762030.083817:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762030.083820:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762030.083829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048817920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762030.083843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762030.083846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762030.083847:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762030.083850:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1479 00000100:00080000:3.0:1789762030.083868:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762030.083872:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762030.083876:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762030.083882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762030.083887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048818048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762030.083894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762030.083897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762030.083899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762030.083902:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1479 00000100:00080000:3.0:1789762030.083904:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762030.083907:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762030.083911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762030.083918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048818176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762030.083966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762030/real 1789762030] req@ffff9811e1a20700 x1876700048817664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762030.083993:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048817664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762030.084001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762030.084004:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762030.084007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048817664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762030.084031:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048817792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762030.084050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762030.084059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762030/real 1789762030] req@ffff9811e1a20000 x1876700048817792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762030.084067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048817792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762030.084072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762030.084075:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762030.084077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048817792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762030.084081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762030.084091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762030.084097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762030/real 1789762030] req@ffff9811e1a23480 x1876700048817920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762030.084105:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048817920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762030.084119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762030.084122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762030.084124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048817920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762030.084128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762030.084142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762030.084148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762030/real 1789762030] req@ffff9811e1a23800 x1876700048818048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762030.084156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048818048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762030.084161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762030.084163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762030.084165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762030.084168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762030.084179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762030.084194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762030/real 1789762030] req@ffff9811e1a23100 x1876700048818176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762030.084202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048818176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762030.084206:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762030.084208:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762030.084214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762030.339538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762030.979609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762031.363599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762032.003592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762032.387559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762033.027621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762033.411613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762034.051696:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762034.435598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762035.075662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762035.203611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762035.203623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762035.203629:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762035.203641:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1484 00000100:00080000:2.0:1789762035.203656:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762035.203665:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762035.203739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048818304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762035.203768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762035.203772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762035.203773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762035.203777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1484 00000100:00080000:2.0:1789762035.203780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762035.203783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762035.203793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762035.203806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048818432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762035.203817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762035.203842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789762035.203845:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048818304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762035.203845:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762035.203852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1484 00000100:00080000:2.0:1789762035.203862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762035.203865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762035.203871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762035.203882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048818560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762035.203893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762035.203912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762035.203914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762035.203918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1484 00000100:00080000:2.0:1789762035.203922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762035.203926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762035.203931:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762035.203960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048818688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762035.203969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762035.203974:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762035.203976:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762035.203982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762035.203986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1484 00000100:00080000:2.0:1789762035.203992:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762035.203996:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762035.204010:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762035.204021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048818816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762035.204070:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762035/real 1789762035] req@ffff9811d8639f80 x1876700048818304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762035.204097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048818304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762035.204148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762035.204155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762035.204162:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048818304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762035.204173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048818432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762035.204209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762035.204244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762035/real 1789762035] req@ffff9811d8639500 x1876700048818432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762035.204278:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048818432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762035.204290:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762035.204295:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762035.204300:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048818432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762035.204342:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762035.204372:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762035.204390:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762035/real 1789762035] req@ffff9811d8639880 x1876700048818560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762035.204407:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048818560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762035.204418:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762035.204423:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762035.204427:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762035.204447:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762035.204555:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762035.204575:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762035/real 1789762035] req@ffff9811d863ad80 x1876700048818688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762035.204593:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048818688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762035.204606:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762035.204610:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762035.204615:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762035.204622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762035.204650:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762035.204666:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762035/real 1789762035] req@ffff9811d8639c00 x1876700048818816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762035.204683:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048818816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762035.204694:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762035.204698:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762035.204703:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048818816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762035.459566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762036.099640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762036.483571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762037.123785:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762037.507599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762038.147609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762038.531551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762039.171895:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762039.555573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762040.195602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762040.323580:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762040.323591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762040.323596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762040.323608:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1489 00000100:00080000:2.0:1789762040.323622:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762040.323631:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762040.323675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048818944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762040.323697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762040.323700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762040.323702:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762040.323705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1489 00000100:00080000:2.0:1789762040.323707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762040.323709:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762040.323718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762040.323727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048819072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762040.323750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048818944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762040.323767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762040.323770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762040.323771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762040.323781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1489 00000100:00080000:2.0:1789762040.323785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762040.323789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762040.323793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762040.323803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048819200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762040.323810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762040.323814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762040.323815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762040.323818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1489 00000100:00080000:2.0:1789762040.323821:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762040.323824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762040.323827:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762040.323832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762040.323842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048819328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762040.323851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762040.323867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762040.323870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762040.323874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1489 00000100:00100000:1.0:1789762040.323888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762040/real 1789762040] req@ffff9811d863b100 x1876700048818944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762040.323892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762040.323899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762040.323909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762040.323926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048819456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762040.323928:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048818944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762040.323958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762040.323963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762040.323966:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048818944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762040.323994:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048819072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762040.324018:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762040.324033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762040/real 1789762040] req@ffff9811d863bb80 x1876700048819072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762040.324044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048819072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762040.324050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762040.324052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762040.324054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048819072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762040.324058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762040.324072:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762040.324079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762040/real 1789762040] req@ffff9811d8639180 x1876700048819200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762040.324087:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048819200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762040.324101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762040.324104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762040.324106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762040.324110:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762040.324126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762040.324137:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762040/real 1789762040] req@ffff9811d8639c00 x1876700048819328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762040.324149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048819328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762040.324158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762040.324161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762040.324165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762040.324171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762040.324190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762040.324197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762040/real 1789762040] req@ffff9811d863ad80 x1876700048819456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762040.324205:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048819456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762040.324210:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762040.324212:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762040.324219:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762040.579537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762041.219631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762041.603566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762042.243654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762042.627618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762043.267653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762043.651557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762044.291639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762044.675613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762045.315587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762045.443585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762045.443600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762045.443605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762045.443619:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1495 00000100:00080000:1.0:1789762045.443633:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762045.443643:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762045.443688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048819584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762045.443714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762045.443719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762045.443721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762045.443727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1495 00000100:00080000:1.0:1789762045.443731:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762045.443735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762045.443741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048819584:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762045.443748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762045.443767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048819712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762045.443782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762045.443787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762045.443789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762045.443795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1495 00000100:00080000:1.0:1789762045.443806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762045.443834:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762045.443843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762045.443849:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762045.443861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048819840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762045.443869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762045.443871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762045.443873:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762045.443876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1495 00000100:00080000:1.0:1789762045.443880:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762045.443883:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762045/real 1789762045] req@ffff9811c92c5f80 x1876700048819584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762045.443885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762045.443895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762045.443907:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048819584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762045.443912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048819968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762045.443925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762045.443931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762045.443940:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762045.443942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762045.443964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762045.443968:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048819584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762045.443972:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1495 00000100:00080000:1.0:1789762045.443978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762045.443982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762045.443987:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762045.443996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048820096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762045.444003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048819712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762045.444029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762045.444039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762045/real 1789762045] req@ffff9811c92c7480 x1876700048819712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762045.444055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048819712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762045.444062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762045.444065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762045.444067:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048819712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762045.444081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762045.444097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762045.444105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762045/real 1789762045] req@ffff9811c92c4e00 x1876700048819840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762045.444114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048819840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762045.444120:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762045.444122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762045.444124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762045.444128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762045.444140:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762045.444146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762045/real 1789762045] req@ffff9811c92c6300 x1876700048819968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762045.444154:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048819968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762045.444160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762045.444162:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762045.444164:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048819968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762045.444176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762045.444188:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762045.444194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762045/real 1789762045] req@ffff9811c92c6a00 x1876700048820096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762045.444212:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048820096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762045.444218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762045.444220:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762045.444222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762045.699550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762046.339633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762046.723589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762047.363588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762047.747536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762048.387650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762048.771574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762049.411595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762049.795532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762050.435616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762050.563627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762050.563655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762050.563659:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762050.563669:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1500 00000100:00080000:1.0:1789762050.563679:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762050.563686:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762050.563739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048820224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762050.563768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762050.563772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762050.563774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762050.563778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1500 00000100:00080000:1.0:1789762050.563796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762050.563800:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762050.563809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762050.563826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048820352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762050.563842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048820224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762050.563846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762050.563858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762050.563860:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762050.563864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1500 00000100:00080000:1.0:1789762050.563867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762050.563871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762050.563876:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762050.563886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048820480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762050.563900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762050.563903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762050.563905:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762050.563908:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1500 00000100:00080000:1.0:1789762050.563920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762050.563943:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762050.563944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762050.563954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762050.563969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048820608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762050.563979:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762050.563982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762050.563983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762050.563986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1500 00000100:00080000:1.0:1789762050.563989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762050.563992:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762050.563996:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762050.564008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048820736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762050.564017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762050/real 1789762050] req@ffff9811c92c4700 x1876700048820224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762050.564089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048820224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762050.564103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762050.564109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762050.564112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048820224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762050.564139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048820352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762050.564159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762050.564169:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762050/real 1789762050] req@ffff9811c92c7800 x1876700048820352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762050.564207:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048820352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762050.564213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762050.564215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762050.564218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048820352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762050.564223:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762050.564237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762050.564245:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762050/real 1789762050] req@ffff9811c92c7100 x1876700048820480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762050.564253:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048820480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762050.564258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762050.564259:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762050.564261:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762050.564265:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762050.564277:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762050.564283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762050/real 1789762050] req@ffff9811c92c6a00 x1876700048820608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762050.564290:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048820608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762050.564304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762050.564306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762050.564309:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762050.564312:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762050.564326:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762050.564340:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762050/real 1789762050] req@ffff9811c92c6300 x1876700048820736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762050.564351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048820736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762050.564357:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762050.564359:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762050.564362:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048820736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762050.819512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762051.459648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762051.843537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762052.483599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762052.867559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762053.507616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762053.891571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762054.531649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762054.915605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762055.555601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762055.683563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762055.683576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762055.683582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762055.683596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1505 00000100:00080000:1.0:1789762055.683632:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762055.683642:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762055.683687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048820864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762055.683720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762055.683723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762055.683725:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762055.683730:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1505 00000100:00080000:1.0:1789762055.683733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762055.683738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762055.683763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762055.683779:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048820864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762055.683789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048820992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762055.683805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762055.683809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762055.683811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762055.683815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1505 00000100:00080000:1.0:1789762055.683819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762055.683824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762055.683830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762055.683842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048821120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762055.683852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762055.683856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762055.683868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762055.683870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762055.683877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1505 00000100:00080000:1.0:1789762055.683882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762055.683887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762055.683894:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762055.683897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762055/real 1789762055] req@ffff9811c92c4e00 x1876700048820864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762055.683911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048821248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762055.683916:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048820864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762055.683938:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762055.683939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762055.683943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762055.683947:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762055.683950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762055.683957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048820864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762055.683957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1505 00000100:00080000:1.0:1789762055.683967:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762055.683983:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762055.683989:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048820992:192.168.203.113@tcp:38:kworker.0 00000008:00100000:1.0:1789762055.683990:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762055.684011:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048821376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762055.684017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762055.684031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762055/real 1789762055] req@ffff9811c92c7480 x1876700048820992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762055.684041:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048820992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762055.684049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762055.684052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762055.684055:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048820992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762055.684059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762055.684070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762055.684076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762055/real 1789762055] req@ffff9811c92c5f80 x1876700048821120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762055.684084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048821120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762055.684089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762055.684090:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762055.684092:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762055.684095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762055.684114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762055.684120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762055/real 1789762055] req@ffff9811c92c4380 x1876700048821248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762055.684128:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048821248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762055.684133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762055.684135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762055.684137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762055.684147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762055.684158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762055.684164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762055/real 1789762055] req@ffff9811c92c4a80 x1876700048821376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762055.684171:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048821376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762055.684176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762055.684178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762055.684180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762055.939622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762056.579610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762056.963595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762057.603585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762057.987581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762058.627591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762059.011598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762059.651546:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762060.035537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762060.675607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762060.803560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762060.803568:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762060.803572:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762060.803580:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1510 00000100:00080000:0.0:1789762060.803590:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762060.803611:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762060.803651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048821504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762060.803684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762060.803688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762060.803690:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762060.803695:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1510 00000100:00080000:0.0:1789762060.803701:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762060.803718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762060.803728:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762060.803728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048821504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762060.803745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048821632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762060.803757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762060.803761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762060.803763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762060.803767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1510 00000100:00080000:0.0:1789762060.803785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762060.803790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762060.803796:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762060.803811:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048821760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762060.803823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762060.803827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789762060.803832:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048821632:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789762060.803841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762060.803848:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1510 00000100:00080000:0.0:1789762060.803855:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762060.803860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762060.803863:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821760:192.168.203.113@tcp:38:kworker.0 00000008:00100000:0.0:1789762060.803870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762060.803888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048821888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762060.803902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762060.803906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762060.803908:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762060.803913:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1510 00000100:00080000:0.0:1789762060.803917:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762060.803920:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821888:192.168.203.113@tcp:8:kworker.0 00000100:00080000:0.0:1789762060.803925:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762060.803932:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762060.803946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048822016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762060.803967:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762061.059591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762061.699576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762062.083524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762062.723632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762063.107572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762063.747580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762063.875895:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762063.875957:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762063.875977:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762063.876036:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762063.876049:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762063.876060:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762063.876093:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789762063.876207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762060/real 1789762063] req@ffff9811c2e2ad80 x1876700048822016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762063.876239:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048822016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762063.876255:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762063.876261:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762063.876266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762063.876274:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762060/real 1789762063] req@ffff9811c2e29880 x1876700048821888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762063.876287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048821888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762063.876296:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762063.876300:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762063.876304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762063.876311:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762060/real 1789762063] req@ffff9811c2e2bb80 x1876700048821632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762063.876325:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048821632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762063.876340:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762063.876350:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762063.876354:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048821632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762063.876361:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762060/real 1789762063] req@ffff9811c2e29180 x1876700048821760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762063.876377:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048821760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762063.876384:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762063.876387:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762063.876390:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048821760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762063.876397:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762060/real 1789762063] req@ffff9811c2e28e00 x1876700048821504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762063.876410:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048821504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762063.876418:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762063.876422:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762063.876425:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048821504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762064.131572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762064.771624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762065.155580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762065.795548:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762065.923702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762065.923712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762065.923717:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762065.923728:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1515 00000100:00080000:3.0:1789762065.923740:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762065.923747:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762065.923787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048822144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762065.923819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762065.923822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762065.923823:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762065.923826:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1515 00000100:00080000:3.0:1789762065.923828:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762065.923830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762065.923839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762065.923848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048822272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762065.923858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762065.923860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762065.923862:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762065.923864:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1515 00000100:00080000:3.0:1789762065.923866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762065.923869:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762065.923874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048822144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789762065.923894:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762065.923909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048822400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762065.923921:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762065.923925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762065.923928:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762065.923932:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1515 00000100:00080000:3.0:1789762065.923937:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762065.923941:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762065.923947:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762065.923962:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762065.923967:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048822528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762065.923980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762065.923984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762065.923986:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762065.923991:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1515 00000100:00080000:3.0:1789762065.923995:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762065.923999:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762065.924005:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762065.924018:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048822656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762065.924023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762065/real 1789762065] req@ffff9811e1a22300 x1876700048822144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762065.924055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048822144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762065.924077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762065.924081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762065.924086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048822144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762065.924095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048822272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762065.924124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762065.924139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762065/real 1789762065] req@ffff9811e1a21180 x1876700048822272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762065.924151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048822272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762065.924160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762065.924164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762065.924168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048822272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762065.924199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762065.924222:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762065.924236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762065/real 1789762065] req@ffff9811e1a22a00 x1876700048822400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762065.924249:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048822400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762065.924258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762065.924269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762065.924272:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762065.924288:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762065.924314:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762065.924335:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762065/real 1789762065] req@ffff9811e1a21c00 x1876700048822528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762065.924351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048822528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762065.924359:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762065.924362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762065.924366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762065.924373:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762065.924397:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762065.924410:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762065/real 1789762065] req@ffff9811e1a21880 x1876700048822656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762065.924422:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048822656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762065.924431:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762065.924434:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762065.924438:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048822656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762066.179574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762066.819589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762067.203533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762067.843604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762068.227541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762068.867598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762069.251527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762069.891590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762070.275539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762070.915606:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762071.043590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762071.043604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762071.043610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762071.043654:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1520 00000100:00080000:1.0:1789762071.043667:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762071.043675:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762071.043714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048822784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762071.043742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762071.043745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762071.043747:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762071.043750:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1520 00000100:00080000:1.0:1789762071.043753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762071.043755:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762071.043764:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762071.043775:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048822912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762071.043784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762071.043786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762071.043788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762071.043790:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1520 00000100:00080000:1.0:1789762071.043792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762071.043798:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048822784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762071.043798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762071.043803:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762071.043813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048823040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762071.043834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762071.043837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762071.043839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762071.043842:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1520 00000100:00080000:1.0:1789762071.043845:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762071.043848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762071.043852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762071.043861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048823168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762071.043868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762071.043870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762071.043871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762071.043873:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1520 00000100:00080000:1.0:1789762071.043876:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762071.043878:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762071.043881:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762071.043884:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762071.043893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048823296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762071.043941:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762071/real 1789762071] req@ffff9811c92c4000 x1876700048822784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762071.043962:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048822784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762071.043974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762071.043985:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762071.043989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048822784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762071.043997:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048822912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762071.044020:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762071.044033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762071/real 1789762071] req@ffff9811c92c6680 x1876700048822912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762071.044047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048822912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762071.044057:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762071.044060:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762071.044064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048822912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762071.044092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762071.044113:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762071.044124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762071/real 1789762071] req@ffff9811c92c5c00 x1876700048823040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762071.044137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048823040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762071.044145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762071.044149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762071.044153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762071.044158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762071.044184:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762071.044195:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762071/real 1789762071] req@ffff9811c92c5880 x1876700048823168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762071.044207:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048823168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762071.044215:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762071.044219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762071.044222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762071.044228:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762071.044244:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762071.044254:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762071/real 1789762071] req@ffff9811c92c5500 x1876700048823296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762071.044266:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048823296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762071.044274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762071.044276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762071.044279:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762071.299531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762071.939575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762072.323550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762072.963605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762073.347551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762073.987574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762074.371547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762075.011610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762075.395524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762076.035585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762076.163573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762076.163600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762076.163606:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762076.163620:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1525 00000100:00080000:1.0:1789762076.163658:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762076.163667:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762076.163714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048823424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762076.163747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762076.163751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762076.163753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762076.163758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1525 00000100:00080000:1.0:1789762076.163762:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762076.163767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762076.163778:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762076.163790:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048823424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762076.163795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048823552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762076.163809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762076.163814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762076.163816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762076.163820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1525 00000100:00080000:1.0:1789762076.163824:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762076.163829:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762076.163834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762076.163847:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048823680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762076.163858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762076.163868:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762076.163870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762076.163873:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762076.163879:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1525 00000100:00080000:1.0:1789762076.163885:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762076.163889:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762076.163895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762076.163908:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048823808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762076.163919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762076.163923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789762076.163927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762076/real 1789762076] req@ffff9811c92c7b80 x1876700048823424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762076.163928:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762076.163937:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1525 00000100:00080000:1.0:1789762076.163944:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762076.163949:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762076.163962:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762076.163977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048823936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762076.163983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048823424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762076.164004:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762076.164009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762076.164015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048823424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762076.164048:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048823552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762076.164073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762076.164084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762076/real 1789762076] req@ffff9811c92c5180 x1876700048823552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762076.164094:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048823552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762076.164100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762076.164102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762076.164105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048823552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762076.164108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762076.164121:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762076.164127:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762076/real 1789762076] req@ffff9811c92c6d80 x1876700048823680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762076.164135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048823680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762076.164151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762076.164153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762076.164155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762076.164160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762076.164174:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762076.164181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762076/real 1789762076] req@ffff9811c92c5500 x1876700048823808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762076.164188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048823808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762076.164194:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762076.164195:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762076.164197:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762076.164201:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762076.164211:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762076.164217:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762076/real 1789762076] req@ffff9811c92c5880 x1876700048823936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762076.164224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048823936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762076.164237:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762076.164239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762076.164249:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048823936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762076.419557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762077.059686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762077.443552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762078.083591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762078.467639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762079.107585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762079.491534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762080.131605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762080.515599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762081.155650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762081.283598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762081.283609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762081.283613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762081.283630:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1530 00000100:00080000:1.0:1789762081.283641:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762081.283649:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762081.283689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048824064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762081.283735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762081.283740:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762081.283743:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762081.283748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1530 00000100:00080000:1.0:1789762081.283754:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762081.283760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762081.283770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762081.283779:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048824064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762081.283785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048824192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762081.283796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762081.283799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762081.283801:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762081.283804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1530 00000100:00080000:1.0:1789762081.283807:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762081.283810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762081.283816:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762081.283830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048824320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762081.283839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762081.283845:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762081.283846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762081.283849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762081.283853:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1530 00000100:00080000:1.0:1789762081.283866:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762081.283872:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762081.283879:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762081.283893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048824448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762081.283898:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762081/real 1789762081] req@ffff9811c92c5c00 x1876700048824064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762081.283916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762081.283920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762081.283922:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762081.283926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048824064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762081.283929:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1530 00000100:00080000:1.0:1789762081.283935:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762081.283940:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789762081.283941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762081.283945:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762081.283947:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762081.283953:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048824064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762081.283958:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048824576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762081.283976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048824192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762081.284006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762081.284019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762081/real 1789762081] req@ffff9811c92c6680 x1876700048824192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762081.284043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048824192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762081.284050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762081.284053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762081.284057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048824192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762081.284064:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762081.284086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762081.284107:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762081/real 1789762081] req@ffff9811c92c4000 x1876700048824320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762081.284120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048824320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762081.284129:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762081.284133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762081.284137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762081.284143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762081.284162:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762081.284171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762081/real 1789762081] req@ffff9811c92c4a80 x1876700048824448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762081.284179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048824448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762081.284184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762081.284187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762081.284190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762081.284194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762081.284208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762081.284219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762081/real 1789762081] req@ffff9811c92c4380 x1876700048824576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762081.284232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048824576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762081.284249:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762081.284253:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762081.284256:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762081.539720:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762082.179688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762082.563601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762083.203617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762083.587625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762084.227652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762084.611566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762085.251598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762085.635591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762086.275653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762086.403614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762086.403626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762086.403646:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762086.403658:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1535 00000100:00080000:1.0:1789762086.403670:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762086.403678:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762086.403729:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048824704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762086.403770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762086.403774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762086.403776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762086.403783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1535 00000100:00080000:1.0:1789762086.403788:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762086.403795:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762086.403805:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762086.403840:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048824704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762086.403853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048824832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762086.403868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762086.403873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762086.403875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762086.403879:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1535 00000100:00080000:1.0:1789762086.403900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762086.403905:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762086.403911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762086.403923:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048824960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762086.403933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762086.403937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762086.403939:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762086.403944:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1535 00000100:00080000:1.0:1789762086.403955:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762086.403961:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762086.403967:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762086.403968:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762086.403996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048825088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762086.404010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762086.404014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762086.404016:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762086.404025:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762086/real 1789762086] req@ffff9811c92c5f80 x1876700048824704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762086.404027:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1535 00000100:00080000:1.0:1789762086.404041:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762086.404051:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762086.404057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048824704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762086.404061:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762086.404087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762086.404092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048825216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762086.404097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762086.404109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048824704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762086.404148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048824832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762086.404176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762086.404190:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762086/real 1789762086] req@ffff9811c92c7480 x1876700048824832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762086.404205:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048824832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762086.404215:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762086.404219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762086.404223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048824832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762086.404240:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762086.404265:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762086.404278:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762086/real 1789762086] req@ffff9811c92c4e00 x1876700048824960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762086.404293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048824960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762086.404300:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762086.404303:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762086.404305:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048824960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762086.404310:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762086.404332:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762086.404362:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762086/real 1789762086] req@ffff9811c92c6300 x1876700048825088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762086.404375:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048825088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762086.404383:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762086.404387:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762086.404391:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762086.404399:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762086.404420:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762086.404429:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762086/real 1789762086] req@ffff9811c92c6a00 x1876700048825216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762086.404488:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048825216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762086.404499:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762086.404503:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762086.404505:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762086.659561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762087.299635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762087.683699:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762088.323658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762088.707593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762089.347585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762089.731609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762090.371692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762090.755557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762091.395661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762091.523586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762091.523597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762091.523602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762091.523613:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1541 00000100:00080000:1.0:1789762091.523637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762091.523644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762091.523690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048825344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762091.523738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762091.523741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762091.523743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762091.523746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1541 00000100:00080000:1.0:1789762091.523749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762091.523751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762091.523761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762091.523774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048825472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762091.523798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762091.523802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762091.523805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762091.523813:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048825344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762091.523815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1541 00000100:00080000:1.0:1789762091.523828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762091.523837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762091.523842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762091.523855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048825600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762091.523863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762091.523868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762091.523870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762091.523875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1541 00000100:00080000:1.0:1789762091.523887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762091.523897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762091.523899:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762091.523908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762091.523925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048825728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762091.523933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762091.523936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762091.523940:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762091.523945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1541 00000100:00080000:1.0:1789762091.523951:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762091.523957:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762091/real 1789762091] req@ffff9811c92c7100 x1876700048825344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762091.523960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762091.523974:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762091.523991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048825856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762091.523995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048825344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762091.524033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762091.524039:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762091.524045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048825344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762091.524070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048825472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762091.524094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762091.524106:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762091/real 1789762091] req@ffff9811c92c7800 x1876700048825472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762091.524119:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048825472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762091.524148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762091.524151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762091.524155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048825472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762091.524162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762091.524182:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762091.524193:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762091/real 1789762091] req@ffff9811c92c4700 x1876700048825600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762091.524205:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048825600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762091.524213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762091.524217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762091.524220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762091.524227:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762091.524244:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762091.524252:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762091/real 1789762091] req@ffff9811c92c6a00 x1876700048825728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762091.524260:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048825728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762091.524265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762091.524267:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762091.524274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762091.524278:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762091.524292:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762091.524298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762091/real 1789762091] req@ffff9811c92c6300 x1876700048825856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762091.524305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048825856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762091.524310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762091.524312:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762091.524314:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048825856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762091.779588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762092.419680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762092.803555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762093.443583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762093.827571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762094.467680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762094.851580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762095.491806:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762095.875566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762096.515734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762096.643631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762096.643641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762096.643644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762096.643653:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1546 00000100:00080000:2.0:1789762096.643663:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762096.643671:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762096.643716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048825984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762096.643740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762096.643743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762096.643745:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762096.643748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1546 00000100:00080000:2.0:1789762096.643750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762096.643752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762096.643760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762096.643768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048826112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762096.643778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762096.643781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762096.643798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762096.643801:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1546 00000100:00080000:2.0:1789762096.643804:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762096.643809:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762096.643814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762096.643823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048826240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762096.643835:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048825984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762096.643842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762096.643850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762096.643853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762096.643859:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1546 00000100:00080000:2.0:1789762096.643864:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762096.643871:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762096.643880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762096.643897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048826368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762096.643905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762096.643908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762096.643910:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762096.643912:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1546 00000100:00080000:2.0:1789762096.643917:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762096.643921:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762096.643925:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762096.643939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762096.643947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048826496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762096.644015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762096/real 1789762096] req@ffff9811d8639880 x1876700048825984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762096.644067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048825984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762096.644076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762096.644080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762096.644083:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048825984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762096.644089:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048826112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762096.644109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762096.644133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762096/real 1789762096] req@ffff9811d8639500 x1876700048826112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762096.644141:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048826112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762096.644147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762096.644149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762096.644151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048826112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762096.644176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762096.644190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762096.644196:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762096/real 1789762096] req@ffff9811d8639f80 x1876700048826240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762096.644203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048826240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762096.644208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762096.644210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762096.644212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762096.644216:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762096.644226:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762096.644232:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762096/real 1789762096] req@ffff9811d863a680 x1876700048826368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762096.644238:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048826368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762096.644248:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762096.644251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762096.644253:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762096.644257:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762096.644270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762096.644276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762096/real 1789762096] req@ffff9811d8638a80 x1876700048826496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762096.644283:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048826496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762096.644288:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762096.644289:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762096.644291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762096.899534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762097.539677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762097.923603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762098.563722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762098.947537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762099.587586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762099.971565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762100.611620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762100.995594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762101.635633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762101.763616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762101.763646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762101.763651:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762101.763665:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1551 00000100:00080000:0.0:1789762101.763683:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762101.763695:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762101.763766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048826624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762101.763818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762101.763860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762101.763879:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762101.763885:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1551 00000100:00080000:0.0:1789762101.763891:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762101.763896:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762101.763899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048826624:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762101.763906:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762101.763922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048826752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762101.763935:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762101.763939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762101.763950:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762101.763955:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1551 00000100:00080000:0.0:1789762101.763960:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762101.763965:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762101.763966:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762101.763970:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762101.763983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048826880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762101.763995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762101.763999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762101.764001:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762101.764005:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1551 00000100:00080000:0.0:1789762101.764009:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762101.764009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762101/real 1789762101] req@ffff9811c2e29c00 x1876700048826624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762101.764014:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762101.764026:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762101.764029:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048826624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762101.764051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048827008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762101.764060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762101.764062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762101.764065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762101.764066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762101.764069:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762101.764071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048826624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762101.764073:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1551 00000100:00080000:0.0:1789762101.764077:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762101.764081:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762101.764086:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762101.764096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048826752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762101.764097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048827136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762101.764126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762101.764143:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762101/real 1789762101] req@ffff9811c2e2a680 x1876700048826752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762101.764158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048826752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762101.764186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762101.764190:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762101.764194:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048826752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762101.764202:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762101.764230:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762101.764270:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762101/real 1789762101] req@ffff9811c2e28700 x1876700048826880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762101.764284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048826880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762101.764294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762101.764298:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762101.764301:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048826880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762101.764320:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762101.764345:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762101.764370:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762101/real 1789762101] req@ffff9811c2e28e00 x1876700048827008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762101.764384:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048827008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762101.764394:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762101.764422:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762101.764426:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762101.764471:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762101.764528:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762101.764539:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762101/real 1789762101] req@ffff9811c2e29180 x1876700048827136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762101.764582:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048827136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762101.764592:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762101.764596:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762101.764600:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762102.019651:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762102.659594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762103.043603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762103.683598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762104.067534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762104.707635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762105.091573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762105.731601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762106.115585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762106.755592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762106.883562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762106.883570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762106.883574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762106.883582:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1556 00000100:00080000:0.0:1789762106.883592:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762106.883597:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762106.883629:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048827264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762106.883653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762106.883656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762106.883657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762106.883660:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1556 00000100:00080000:0.0:1789762106.883678:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762106.883682:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762106.883690:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762106.883697:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048827264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762106.883702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048827392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762106.883719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762106.883721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762106.883723:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762106.883726:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1556 00000100:00080000:0.0:1789762106.883729:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762106.883731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762106.883733:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762106.883742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048827520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762106.883752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762106.883755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762106.883757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762106.883762:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1556 00000100:00080000:0.0:1789762106.883765:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762106.883772:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762106.883772:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762106.883781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762106.883795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048827648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762106.883810:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762106/real 1789762106] req@ffff9811c2e2bb80 x1876700048827264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762106.883825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762106.883829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762106.883831:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762106.883835:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048827264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762106.883836:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1556 00000100:00080000:0.0:1789762106.883842:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762106.883849:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789762106.883850:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789762106.883854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762106.883855:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762106.883861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048827264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762106.883871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048827776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762106.883887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048827392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762106.883915:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762106.883927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762106/real 1789762106] req@ffff9811c2e29880 x1876700048827392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762106.883936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048827392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762106.883950:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762106.883952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762106.883955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048827392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762106.883959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762106.883972:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762106.883980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762106/real 1789762106] req@ffff9811c2e2ad80 x1876700048827520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762106.883998:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048827520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762106.884004:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762106.884006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762106.884008:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762106.884012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762106.884025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762106.884031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762106/real 1789762106] req@ffff9811c2e2b100 x1876700048827648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762106.884038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048827648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762106.884044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762106.884045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762106.884047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762106.884058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762106.884068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762106.884074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762106/real 1789762106] req@ffff9811c2e2aa00 x1876700048827776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762106.884080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048827776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762106.884094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762106.884096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762106.884098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048827776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762107.139556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762107.779595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762108.163540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762108.803589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762109.187582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762109.827624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762110.211581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762110.851721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762111.235577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762111.875578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762112.003644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762112.003654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762112.003658:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762112.003668:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1561 00000100:00080000:1.0:1789762112.003678:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762112.003685:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762112.003722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048827904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762112.003750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762112.003755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762112.003757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762112.003762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1561 00000100:00080000:1.0:1789762112.003767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762112.003773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762112.003785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762112.003808:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048827904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762112.003816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048828032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762112.003831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762112.003849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762112.003851:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762112.003855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1561 00000100:00080000:1.0:1789762112.003859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762112.003862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762112.003866:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762112.003867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762112.003882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048828160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762112.003889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762112.003892:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762112.003893:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762112.003897:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1561 00000100:00080000:1.0:1789762112.003900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762112.003902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762112.003906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762112.003908:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762112/real 1789762112] req@ffff9811c92c4e00 x1876700048827904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762112.003916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048828288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762112.003926:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762112.003928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762112.003930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762112.003933:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048827904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762112.003936:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1561 00000100:00080000:1.0:1789762112.003941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762112.003947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789762112.003953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762112.003957:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762112.003959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762112.003965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048827904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762112.003970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048828416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762112.003990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048828032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762112.004008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762112.004016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762112/real 1789762112] req@ffff9811c92c7480 x1876700048828032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762112.004033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048828032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762112.004040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762112.004042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762112.004045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048828032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762112.004049:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762112.004063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762112.004069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762112/real 1789762112] req@ffff9811c92c5f80 x1876700048828160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762112.004078:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048828160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762112.004083:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762112.004085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762112.004087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762112.004090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762112.004101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762112.004107:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762112/real 1789762112] req@ffff9811c92c4380 x1876700048828288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762112.004115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048828288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762112.004129:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762112.004131:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762112.004134:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762112.004137:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762112.004150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762112.004156:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762112/real 1789762112] req@ffff9811c92c4a80 x1876700048828416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762112.004163:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048828416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762112.004169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762112.004171:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762112.004173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762112.259593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762112.899548:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762113.283611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762113.923564:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762114.307532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762114.947712:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762115.331539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762115.971645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762116.355550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762116.995642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762117.123600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762117.123619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762117.123623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762117.123635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1566 00000100:00080000:2.0:1789762117.123646:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762117.123655:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762117.123701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048828544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762117.123742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762117.123747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762117.123750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762117.123755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1566 00000100:00080000:2.0:1789762117.123760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762117.123763:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048828544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762117.123769:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762117.123781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762117.123798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048828672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762117.123813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762117.123817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1789762117.123821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762117.123822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762117.123830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1566 00000100:00080000:2.0:1789762117.123835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762117.123840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762117.123846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762117.123858:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762117/real 1789762117] req@ffff9811d863aa00 x1876700048828544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762117.123862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048828800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762117.123877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048828544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762117.123893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762117.123895:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762117.123902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762117.123903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789762117.123908:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762117.123910:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048828544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762117.123915:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1566 00000100:00080000:2.0:1789762117.123923:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762117.123929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762117.123935:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762117.123943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048828672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762117.123951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048828928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762117.123965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762117.123967:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762117.123972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762117.123975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762117.123982:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1566 00000100:00100000:0.0:1789762117.123983:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762117/real 1789762117] req@ffff9811d8638000 x1876700048828672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762117.123986:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762117.123996:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762117.124001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048828672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762117.124002:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762117.124014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762117.124017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762117.124023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048828672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762117.124024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048829056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762117.124029:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762117.124048:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762117.124056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762117/real 1789762117] req@ffff9811d863b800 x1876700048828800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762117.124065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048828800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762117.124071:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762117.124073:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762117.124075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762117.124091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762117.124117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762117.124130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762117/real 1789762117] req@ffff9811d863b480 x1876700048828928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762117.124155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048828928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762117.124165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762117.124170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762117.124185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048828928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762117.124194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762117.124221:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762117.124236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762117/real 1789762117] req@ffff9811d8638380 x1876700048829056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762117.124244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048829056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762117.124250:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762117.124253:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762117.124255:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762117.379620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762118.019724:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762118.403525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762119.043634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762119.427545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762120.067651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762120.451568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762121.091723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762121.475556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762122.115707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762122.243626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762122.243638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762122.243642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762122.243653:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1571 00000100:00080000:0.0:1789762122.243665:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762122.243694:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762122.243734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048829184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762122.243765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762122.243768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762122.243769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762122.243772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1571 00000100:00080000:0.0:1789762122.243776:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762122.243778:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762122.243788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762122.243797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048829312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762122.243807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762122.243813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762122.243816:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762122.243818:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048829184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762122.243823:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1571 00000100:00080000:0.0:1789762122.243831:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762122.243836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762122.243839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762122.243849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048829440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762122.243857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762122.243860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762122.243866:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762122.243870:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1571 00000100:00080000:0.0:1789762122.243874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762122.243877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762122.243889:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762122.243895:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762122.243903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048829568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762122.243913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762122.243916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762122.243918:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762122.243922:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1571 00000100:00080000:0.0:1789762122.243925:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762122.243928:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762122.243937:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762122.243944:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762122/real 1789762122] req@ffff9811c2e28380 x1876700048829184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762122.243946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048829696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762122.244022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048829184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762122.244031:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762122.244034:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762122.244037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048829184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762122.244060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048829312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762122.244079:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762122.244088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762122/real 1789762122] req@ffff9811c2e28a80 x1876700048829312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762122.244097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048829312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762122.244103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762122.244106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762122.244108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048829312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762122.244112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762122.244125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762122.244131:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762122/real 1789762122] req@ffff9811c2e29f80 x1876700048829440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762122.244138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048829440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762122.244152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762122.244155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762122.244157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762122.244162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762122.244176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762122.244183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762122/real 1789762122] req@ffff9811c2e2b800 x1876700048829568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762122.244190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048829568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762122.244205:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762122.244207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762122.244209:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762122.244214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762122.244226:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762122.244233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762122/real 1789762122] req@ffff9811c2e2b480 x1876700048829696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762122.244240:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048829696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762122.244258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762122.244260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762122.244268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048829696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762122.499620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762123.139799:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762123.525296:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762124.163701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762124.547679:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762125.187696:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762125.571562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762126.211677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762126.595605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762127.235699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762127.363563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762127.363574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762127.363578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762127.363590:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1576 00000100:00080000:2.0:1789762127.363601:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762127.363608:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762127.363662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048829824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762127.363686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762127.363689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762127.363690:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762127.363693:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1576 00000100:00080000:2.0:1789762127.363695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762127.363697:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762127.363707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762127.363717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048829952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762127.363727:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048829824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762127.363728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762127.363735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762127.363738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762127.363742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1576 00000100:00080000:2.0:1789762127.363744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762127.363764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762127.363769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762127.363779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048830080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762127.363786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762127.363789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762127.363790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762127.363792:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1576 00000100:00080000:2.0:1789762127.363795:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762127.363797:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762127.363801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762127.363811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048830208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762127.363817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762127.363820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762127.363821:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762127.363824:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1576 00000100:00080000:2.0:1789762127.363826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762127.363828:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762127.363831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762127.363839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048830336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762127.363971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048829952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762127.364056:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762127.364082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762127.364126:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762127.619539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762128.259655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762128.643587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762129.283745:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762129.667621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762130.307630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762130.435698:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762130.435736:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762130.435750:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762130.435780:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762130.435786:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762130.435793:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762130.435800:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789762130.435822:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762127/real 1789762130] req@ffff9811d8638700 x1876700048829952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762130.435844:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048829952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762130.435854:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762130.435858:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762130.435861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048829952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762130.435867:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762127/real 1789762130] req@ffff9811d8638e00 x1876700048830080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762130.435879:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048830080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762130.435888:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762130.435892:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762130.435896:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762130.435903:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762127/real 1789762130] req@ffff9811d8638380 x1876700048830208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762130.435916:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048830208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762130.435946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762130.435951:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762130.435955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762130.435963:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762127/real 1789762130] req@ffff9811d863b480 x1876700048830336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762130.435978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048830336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762130.435987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762130.435991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762130.435994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762130.436001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762127/real 1789762130] req@ffff9811d863a300 x1876700048829824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762130.436016:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048829824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762130.436025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762130.436028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762130.436032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048829824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762130.691567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762131.331641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762131.715537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762132.355712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762132.483597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762132.483607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762132.483610:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762132.483618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1582 00000100:00080000:1.0:1789762132.483628:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762132.483634:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762132.483665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048830464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762132.483693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762132.483696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762132.483698:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762132.483700:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1582 00000100:00080000:1.0:1789762132.483703:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762132.483705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762132.483726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762132.483737:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048830592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762132.483747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762132.483750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762132.483751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762132.483754:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1582 00000100:00080000:1.0:1789762132.483757:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762132.483759:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762132.483763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762132.483767:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048830464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762132.483773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048830720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762132.483784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762132.483786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762132.483788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762132.483790:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1582 00000100:00080000:1.0:1789762132.483793:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762132.483795:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762132.483799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762132.483807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048830848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762132.483814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762132.483817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762132.483824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762132.483829:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1582 00000100:00080000:1.0:1789762132.483835:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762132.483840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762132.483847:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762132.483849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762132.483866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048830976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762132.483898:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762132/real 1789762132] req@ffff9811c92c4000 x1876700048830464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762132.483932:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048830464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762132.483945:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762132.483950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762132.483962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048830464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762132.483972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048830592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762132.484002:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762132.484018:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762132/real 1789762132] req@ffff9811c92c6680 x1876700048830592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762132.484032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048830592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762132.484042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762132.484046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762132.484050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048830592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762132.484077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762132.484100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762132.484112:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762132/real 1789762132] req@ffff9811c92c5c00 x1876700048830720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762132.484126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048830720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762132.484152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762132.484156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762132.484160:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762132.484182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762132.484209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762132.484223:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762132/real 1789762132] req@ffff9811c92c5880 x1876700048830848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762132.484237:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048830848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762132.484246:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762132.484250:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762132.484253:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762132.484261:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762132.484283:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762132.484295:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762132/real 1789762132] req@ffff9811c92c5500 x1876700048830976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762132.484308:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048830976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762132.484318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762132.484322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762132.484343:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048830976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762132.739523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762133.379688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762133.763513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762134.403653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762134.787504:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762135.427615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762135.811502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762136.451645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762136.835506:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762137.475624:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762137.603563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762137.603572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762137.603575:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762137.603584:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1587 00000100:00080000:1.0:1789762137.603604:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762137.603610:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762137.603646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048831104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762137.603684:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762137.603687:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762137.603689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762137.603692:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1587 00000100:00080000:1.0:1789762137.603694:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762137.603696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762137.603703:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762137.603738:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048831104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762137.603740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048831232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762137.603755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762137.603757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762137.603759:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762137.603762:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1587 00000100:00080000:1.0:1789762137.603765:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762137.603767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762137.603772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762137.603781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048831360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762137.603788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762137.603790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762137.603792:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762137.603794:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1587 00000100:00080000:1.0:1789762137.603797:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762137.603800:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762137.603803:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762137.603808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762137.603826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048831488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762137.603834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762137.603836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762137.603838:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762137.603845:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1587 00000100:00080000:1.0:1789762137.603849:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762137.603853:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762137/real 1789762137] req@ffff9811c92c6d80 x1876700048831104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762137.603855:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762137.603863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762137.603874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048831616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762137.603877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048831104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762137.603889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762137.603896:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762137.603901:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048831104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762137.603931:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048831232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762137.603948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762137.603956:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762137/real 1789762137] req@ffff9811c92c5180 x1876700048831232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762137.603964:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048831232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762137.603970:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762137.603971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762137.603973:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048831232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762137.603977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762137.603997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762137.604004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762137/real 1789762137] req@ffff9811c92c7b80 x1876700048831360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762137.604012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048831360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762137.604017:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762137.604019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762137.604021:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762137.604025:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762137.604035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762137.604041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762137/real 1789762137] req@ffff9811c92c5500 x1876700048831488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762137.604049:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048831488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762137.604054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762137.604055:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762137.604057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762137.604060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762137.604069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762137.604084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762137/real 1789762137] req@ffff9811c92c5880 x1876700048831616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762137.604101:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048831616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762137.604107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762137.604108:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762137.604111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048831616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762137.859516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762138.499601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762138.883506:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762139.523611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762139.907633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762140.547635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762140.931523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762141.571625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762141.955560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762142.595611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762142.723586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762142.723623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762142.723628:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762142.723637:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1592 00000100:00080000:2.0:1789762142.723649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762142.723657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762142.723693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048831744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762142.723725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762142.723729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762142.723731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762142.723736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1592 00000100:00080000:2.0:1789762142.723740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762142.723744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762142.723752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762142.723766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048831872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762142.723779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762142.723781:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048831744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762142.723786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762142.723790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762142.723795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1592 00000100:00080000:2.0:1789762142.723799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762142.723804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762142.723810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762142.723821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048832000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762142.723834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762142.723839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762142.723840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762142.723844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762142.723848:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1592 00000100:00080000:2.0:1789762142.723853:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762142.723856:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762142.723861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762142.723870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048832128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762142.723877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762142.723881:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762142/real 1789762142] req@ffff9811d863b800 x1876700048831744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762142.723882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762142.723885:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762142.723892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1592 00000100:00080000:2.0:1789762142.723897:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762142.723902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762142.723905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762142.723912:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048831744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762142.723916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048832256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762142.723937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762142.723943:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762142.723946:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048831744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762142.723965:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048831872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762142.723981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762142.723989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762142/real 1789762142] req@ffff9811d8638000 x1876700048831872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762142.723997:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048831872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762142.724009:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762142.724012:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762142.724014:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048831872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762142.724019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762142.724032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762142.724039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762142/real 1789762142] req@ffff9811d863aa00 x1876700048832000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762142.724046:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048832000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762142.724052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762142.724053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762142.724055:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762142.724059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762142.724070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762142.724075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762142/real 1789762142] req@ffff9811d8638a80 x1876700048832128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762142.724083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048832128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762142.724087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762142.724103:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762142.724113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762142.724120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762142.724141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762142.724153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762142/real 1789762142] req@ffff9811d863a680 x1876700048832256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762142.724166:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048832256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762142.724172:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762142.724175:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762142.724177:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762142.979531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762143.619622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762144.003565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762144.643634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762145.027519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762145.667591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762146.051575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762146.691632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762147.075589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762147.715580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762147.843589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762147.843600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762147.843604:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762147.843614:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1597 00000100:00080000:1.0:1789762147.843623:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762147.843630:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762147.843669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048832384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762147.843699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762147.843702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762147.843704:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762147.843715:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1597 00000100:00080000:1.0:1789762147.843719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762147.843724:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762147.843734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762147.843758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048832384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762147.843766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048832512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762147.843777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762147.843780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762147.843791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762147.843795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1597 00000100:00080000:1.0:1789762147.843810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762147.843814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762147.843818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762147.843820:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762147.843830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048832640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762147.843839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762147.843841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762147.843842:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762147.843845:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1597 00000100:00080000:1.0:1789762147.843848:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762147.843851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762147.843855:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762147.843862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048832768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762147.843867:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762147/real 1789762147] req@ffff9811c92c5c00 x1876700048832384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762147.843873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762147.843882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762147.843885:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762147.843896:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1597 00000100:00100000:0.0:1789762147.843901:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048832384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762147.843902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762147.843909:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762147.843915:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762147.843922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762147.843927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048832896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762147.843932:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762147.843942:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048832384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762147.843968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048832512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762147.844003:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762147.844017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762147/real 1789762147] req@ffff9811c92c6680 x1876700048832512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762147.844032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048832512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762147.844042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762147.844045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762147.844070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048832512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762147.844077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762147.844097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762147.844109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762147/real 1789762147] req@ffff9811c92c4000 x1876700048832640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762147.844121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048832640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762147.844130:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762147.844133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762147.844136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762147.844143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762147.844160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762147.844170:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762147/real 1789762147] req@ffff9811c92c4a80 x1876700048832768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762147.844194:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048832768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762147.844202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762147.844205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762147.844209:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762147.844215:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762147.844233:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762147.844244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762147/real 1789762147] req@ffff9811c92c4380 x1876700048832896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762147.844257:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048832896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762147.844265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762147.844268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762147.844271:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048832896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762148.099574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762148.739601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762149.123580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762149.763625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762150.147532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762150.787662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762151.171539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762151.811664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762152.195554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762152.835626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762152.963545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762152.963558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762152.963563:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762152.963573:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1602 00000100:00080000:1.0:1789762152.963585:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762152.963595:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762152.963656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048833024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762152.963688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762152.963693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762152.963695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762152.963699:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1602 00000100:00080000:1.0:1789762152.963708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762152.963714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762152.963727:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762152.963728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048833024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762152.963749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048833152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762152.963768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762152.963771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762152.963772:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762152.963776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1602 00000100:00080000:1.0:1789762152.963781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762152.963784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762152.963789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762152.963798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048833280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762152.963813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762152.963816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762152.963817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762152.963819:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1602 00000800:00000100:2.0:1789762152.963824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762152.963825:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762152.963833:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762152.963840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762152.963849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048833408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762152.963855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762152.963857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762152.963859:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762152.963861:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1602 00000100:00080000:1.0:1789762152.963869:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762152.963872:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762152.963876:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762152.963890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048833536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762152.963892:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762152/real 1789762152] req@ffff9811c92c5f80 x1876700048833024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762152.963946:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048833024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762152.963959:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762152.963963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762152.963967:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048833024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762152.963996:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048833152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762152.964035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762152.964052:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762152/real 1789762152] req@ffff9811c92c7480 x1876700048833152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762152.964067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048833152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762152.964077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762152.964080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762152.964084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048833152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762152.964092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762152.964116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762152.964127:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762152/real 1789762152] req@ffff9811c92c4e00 x1876700048833280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762152.964139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048833280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762152.964170:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762152.964177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762152.964182:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762152.964188:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762152.964215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762152.964232:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762152/real 1789762152] req@ffff9811c92c6300 x1876700048833408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762152.964242:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048833408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762152.964249:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762152.964251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762152.964254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762152.964258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762152.964275:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762152.964289:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762152/real 1789762152] req@ffff9811c92c6a00 x1876700048833536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762152.964300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048833536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762152.964318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762152.964321:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762152.964329:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762153.219559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762153.859602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762154.243576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762154.883646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762155.267532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762155.907638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762156.291543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762156.931652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762157.315559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762157.955686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762158.083622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762158.083632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762158.083636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762158.083645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1607 00000100:00080000:2.0:1789762158.083655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762158.083661:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762158.083690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048833664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762158.083714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762158.083716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762158.083718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762158.083721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1607 00000100:00080000:2.0:1789762158.083723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762158.083725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762158.083737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762158.083746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048833792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762158.083757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762158.083761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762158.083764:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762158.083765:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048833664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762158.083770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1607 00000100:00080000:2.0:1789762158.083779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762158.083799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762158.083804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762158.083815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048833920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762158.083823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762158.083827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762158.083829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762158.083833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1607 00000100:00080000:2.0:1789762158.083836:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762158.083838:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762158.083843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762158.083850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048834048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762158.083856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762158.083870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762158.083873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762158.083875:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762158.083879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1607 00000100:00080000:2.0:1789762158.083883:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762158.083887:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762158.083891:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762158.083901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048834176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762158.083929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762158/real 1789762158] req@ffff9811d8639f80 x1876700048833664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762158.083962:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048833664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762158.083972:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762158.083987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762158.083990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048833664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762158.084022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048833792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762158.084044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762158.084054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762158/real 1789762158] req@ffff9811d8639500 x1876700048833792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762158.084063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048833792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762158.084069:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762158.084071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762158.084074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048833792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762158.084078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762158.084089:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762158.084096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762158/real 1789762158] req@ffff9811d8639880 x1876700048833920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762158.084103:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048833920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762158.084118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762158.084121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762158.084123:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048833920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762158.084128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762158.084142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762158.084149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762158/real 1789762158] req@ffff9811d863ad80 x1876700048834048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762158.084157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048834048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762158.084162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762158.084164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762158.084165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762158.084169:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762158.084180:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762158.084186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762158/real 1789762158] req@ffff9811d8639c00 x1876700048834176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762158.084193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048834176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762158.084198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762158.084199:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762158.084206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762158.339525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762158.979597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762159.363586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762160.003664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762160.387544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762161.027646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762161.411624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762162.051631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762162.435539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762163.075601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762163.203586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762163.203596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762163.203607:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762163.203615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1612 00000100:00080000:1.0:1789762163.203625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762163.203632:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762163.203671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048834304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762163.203700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762163.203703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762163.203705:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762163.203708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1612 00000100:00080000:1.0:1789762163.203711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762163.203714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762163.203723:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762163.203752:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048834304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762163.203758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048834432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762163.203774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762163.203779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762163.203781:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762163.203784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1612 00000100:00080000:1.0:1789762163.203788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762163.203792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762163.203796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762163.203806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048834560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762163.203813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762163.203816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762163.203820:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762163.203821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762163.203825:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1612 00000100:00080000:1.0:1789762163.203829:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762163.203832:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762163.203835:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762163.203845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048834688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762163.203853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762163.203865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762163.203870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762163.203871:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762163/real 1789762163] req@ffff9811c92c4700 x1876700048834304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762163.203887:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1612 00000100:00080000:1.0:1789762163.203895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762163.203900:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048834304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762163.203902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762163.203911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762163.203920:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762163.203933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048834816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762163.203948:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762163.203957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048834304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762163.204027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048834432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762163.204052:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762163.204066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762163/real 1789762163] req@ffff9811c92c7800 x1876700048834432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762163.204081:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048834432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762163.204101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762163.204105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762163.204109:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048834432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762163.204116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762163.204150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762163.204163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762163/real 1789762163] req@ffff9811c92c7100 x1876700048834560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762163.204178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048834560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762163.204188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762163.204192:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762163.204196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762163.204203:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762163.204222:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762163.204233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762163/real 1789762163] req@ffff9811c92c6a00 x1876700048834688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762163.204247:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048834688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762163.204256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762163.204260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762163.204263:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762163.204270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762163.204286:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762163.204297:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762163/real 1789762163] req@ffff9811c92c6300 x1876700048834816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762163.204326:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048834816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762163.204335:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762163.204339:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762163.204343:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048834816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762163.459552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762164.099573:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762164.483539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762165.123676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762165.507559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762166.147589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762166.531559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762167.171650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762167.555628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762168.195658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762168.323600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762168.323612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762168.323617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762168.323627:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1617 00000100:00080000:0.0:1789762168.323639:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762168.323647:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762168.323687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048834944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762168.323718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762168.323722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762168.323724:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762168.323728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1617 00000100:00080000:0.0:1789762168.323731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762168.323734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762168.323742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762168.323774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048835072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762168.323787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762168.323790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762168.323792:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762168.323797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1617 00000100:00080000:0.0:1789762168.323801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762168.323805:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762168.323810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762168.323818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048835200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762168.323828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762168.323831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762168.323833:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762168.323840:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1617 00000100:00080000:0.0:1789762168.323843:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762168.323843:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048834944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762168.323848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762168.323852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762168.323866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048835328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762168.323876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762168.323880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762168.323882:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762168.323895:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1617 00000100:00080000:0.0:1789762168.323919:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762168.323926:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762168.323933:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762168.323953:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762168.323954:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048835456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762168.324008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762168/real 1789762168] req@ffff9811c2e29500 x1876700048834944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762168.324030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048834944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762168.324041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762168.324045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762168.324050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048834944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762168.324057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048835072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762168.324077:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762168.324088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762168/real 1789762168] req@ffff9811c2e28000 x1876700048835072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762168.324100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048835072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762168.324108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762168.324112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762168.324115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048835072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762168.324138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762168.324155:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762168.324163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762168/real 1789762168] req@ffff9811c2e2a300 x1876700048835200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762168.324190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048835200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762168.324198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762168.324201:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762168.324204:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762168.324209:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762168.324227:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762168.324263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762168/real 1789762168] req@ffff9811c2e2b480 x1876700048835328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762168.324275:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048835328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762168.324283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762168.324285:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762168.324288:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762168.324293:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762168.324309:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762168.324317:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762168/real 1789762168] req@ffff9811c2e2b800 x1876700048835456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762168.324327:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048835456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762168.324346:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762168.324349:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762168.324352:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762168.579542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762169.219575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762169.603559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762170.243594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762170.627571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762171.267567:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762171.651545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762172.291629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762172.675584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762173.315656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762173.443598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762173.443609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762173.443613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762173.443625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1623 00000100:00080000:0.0:1789762173.443638:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762173.443647:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762173.443710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048835584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762173.443733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762173.443736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762173.443749:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762173.443769:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1623 00000100:00080000:0.0:1789762173.443772:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762173.443775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762173.443786:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762173.443798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048835712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762173.443809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762173.443816:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048835584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762173.443829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762173.443841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762173.443845:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1623 00000100:00080000:0.0:1789762173.443849:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762173.443853:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762173.443858:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762173.443915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048835840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762173.443923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762173.443925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762173.443927:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762173.443930:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1623 00000100:00080000:0.0:1789762173.443933:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762173.443936:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762173.443940:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762173.443948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048835968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762173.443955:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762173.443960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762173.443963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762173.443980:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762173.443983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1623 00000100:00080000:0.0:1789762173.443986:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762173.443991:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762173.443999:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762173.444005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048836096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762173.444149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762173/real 1789762173] req@ffff9811c2e29f80 x1876700048835584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762173.444178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048835584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762173.444191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762173.444194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762173.444198:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048835584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762173.444226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048835712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762173.444271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762173.444281:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762173/real 1789762173] req@ffff9811c2e28a80 x1876700048835712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762173.444290:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048835712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762173.444313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762173.444316:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762173.444318:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048835712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762173.444335:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762173.444350:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762173.444357:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762173/real 1789762173] req@ffff9811c2e28380 x1876700048835840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762173.444364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048835840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762173.444379:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762173.444382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762173.444384:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762173.444389:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762173.444402:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762173.444409:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762173/real 1789762173] req@ffff9811c2e2aa00 x1876700048835968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762173.444421:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048835968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762173.444426:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762173.444428:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762173.444435:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048835968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762173.444440:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762173.444454:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762173.444503:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762173/real 1789762173] req@ffff9811c2e2b100 x1876700048836096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762173.444530:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048836096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762173.444536:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762173.444538:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762173.444541:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762173.699623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762174.339649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762174.723548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762175.363665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762175.747575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762176.387662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762176.771559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762177.411594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762177.795528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762178.435646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762178.563616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762178.563645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762178.563650:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762178.563663:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1628 00000100:00080000:2.0:1789762178.563677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762178.563687:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762178.563735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048836224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762178.563778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762178.563783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762178.563785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762178.563790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1628 00000100:00080000:2.0:1789762178.563818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762178.563873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762178.563874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048836224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789762178.563881:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762178.563894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048836352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762178.563904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762178.563907:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762178.563909:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762178.563912:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1628 00000100:00080000:2.0:1789762178.563915:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762178.563918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762178.563938:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762178.563955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048836480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762178.563965:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762178.563983:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762178.563989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762178.563991:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762178.563998:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1628 00000100:00080000:2.0:1789762178.564004:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762178.564009:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762178.564016:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762178.564031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048836608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762178.564053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762178/real 1789762178] req@ffff9811d8639180 x1876700048836224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762178.564067:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762178.564081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762178.564087:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762178.564102:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1628 00000100:00080000:2.0:1789762178.564109:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762178.564116:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762178.564123:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762178.564130:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048836224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762178.564144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048836736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762178.564182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762178.564194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762178.564202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048836224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762178.564237:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048836352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762178.564274:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762178.564305:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762178/real 1789762178] req@ffff9811d863bb80 x1876700048836352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762178.564315:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048836352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762178.564321:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762178.564323:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762178.564326:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048836352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762178.564329:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762178.564350:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762178.564357:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762178/real 1789762178] req@ffff9811d863b100 x1876700048836480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762178.564365:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048836480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762178.564371:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762178.564373:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762178.564376:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762178.564382:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762178.564404:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762178.564415:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762178/real 1789762178] req@ffff9811d8639c00 x1876700048836608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762178.564428:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048836608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762178.564436:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762178.564439:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762178.564443:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762178.564449:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762178.564471:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762178.564482:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762178/real 1789762178] req@ffff9811d863ad80 x1876700048836736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762178.564501:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048836736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762178.564524:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762178.564527:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762178.564529:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048836736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762178.819559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762179.459590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762179.843546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762180.483587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762180.867522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762181.507634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762181.891533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762182.531718:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762182.915604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762183.555817:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762183.683622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762183.683637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762183.683644:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762183.683659:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1633 00000100:00080000:1.0:1789762183.683676:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762183.683687:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762183.683763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048836864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762183.683827:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762183.683833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762183.683836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762183.683868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1633 00000100:00080000:1.0:1789762183.683876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762183.683884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762183.683901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762183.683904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048836864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762183.683925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048836992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762183.683946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762183.683951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762183.683954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762183.683960:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1633 00000100:00080000:1.0:1789762183.683965:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762183.683970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762183.683977:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762183.683994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048837120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762183.684007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762183.684016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762183.684019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762183.684025:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762183.684032:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1633 00000100:00080000:1.0:1789762183.684037:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762183.684043:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762183.684051:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762183.684067:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048837248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762183.684092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762183/real 1789762183] req@ffff9811c92c4e00 x1876700048836864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762183.684105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762183.684116:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762183.684119:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762183.684128:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1633 00000100:00100000:0.0:1789762183.684136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048836864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762183.684137:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762183.684153:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789762183.684166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762183.684167:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762183.684178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762183.684185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048836864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762183.684194:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048837376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762183.684229:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048836992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762183.684262:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762183.684279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762183/real 1789762183] req@ffff9811c92c7480 x1876700048836992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762183.684293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048836992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762183.684323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762183.684328:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762183.684333:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048836992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762183.684340:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762183.684366:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762183.684402:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762183/real 1789762183] req@ffff9811c92c5f80 x1876700048837120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762183.684417:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048837120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762183.684427:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762183.684431:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762183.684435:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762183.684442:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762183.684536:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762183.684552:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762183/real 1789762183] req@ffff9811c92c4380 x1876700048837248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762183.684567:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048837248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762183.684577:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762183.684581:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762183.684595:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762183.684603:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762183.684632:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762183.684646:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762183/real 1789762183] req@ffff9811c92c4a80 x1876700048837376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762183.684660:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048837376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762183.684670:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762183.684674:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762183.684677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762183.939542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762184.579616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762184.963588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762185.603668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762185.987537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762186.627605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762187.011593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762187.651656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762188.035554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762188.675632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762188.803645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762188.803657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762188.803668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762188.803680:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1638 00000100:00080000:1.0:1789762188.803693:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762188.803701:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762188.803756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048837504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762188.803783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762188.803786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762188.803787:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762188.803790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1638 00000100:00080000:1.0:1789762188.803807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762188.803811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762188.803821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762188.803834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048837632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762188.803839:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048837504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762188.803857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762188.803860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762188.803862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762188.803868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1638 00000100:00080000:1.0:1789762188.803872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762188.803877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762188.803883:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762188.803896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048837760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762188.803908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762188.803912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762188.803913:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762188.803918:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762188.803922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1638 00000100:00080000:1.0:1789762188.803928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762188.803933:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762188.803941:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762188.803957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048837888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762188.803966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762188/real 1789762188] req@ffff9811c92c4000 x1876700048837504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762188.803994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762188.803996:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048837504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762188.804001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762188.804006:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762188.804013:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762188.804015:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1638 00000100:00080000:0.0:1789762188.804019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762188.804024:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762188.804025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048837504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762188.804034:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762188.804045:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762188.804060:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048838016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762188.804077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048837632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762188.804117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762188.804131:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762188/real 1789762188] req@ffff9811c92c6680 x1876700048837632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762188.804142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048837632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762188.804150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762188.804152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762188.804155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048837632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762188.804161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762188.804193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762188.804201:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762188/real 1789762188] req@ffff9811c92c5c00 x1876700048837760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762188.804221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048837760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762188.804228:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762188.804232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762188.804234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762188.804238:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762188.804262:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762188.804269:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762188/real 1789762188] req@ffff9811c92c5880 x1876700048837888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762188.804284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048837888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762188.804290:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762188.804292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762188.804294:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048837888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762188.804298:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762188.804319:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762188.804338:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762188/real 1789762188] req@ffff9811c92c5500 x1876700048838016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762188.804346:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048838016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762188.804355:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762188.804357:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762188.804360:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762189.059575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762189.699631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762190.083577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762190.723617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762191.107547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762191.747600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762192.131534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762192.771655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762193.155567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762193.795629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762193.923618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762193.923628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762193.923632:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762193.923643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1643 00000100:00080000:1.0:1789762193.923655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762193.923671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762193.923734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048838144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762193.923764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762193.923767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762193.923769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762193.923772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1643 00000100:00080000:1.0:1789762193.923774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762193.923778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762193.923806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762193.923818:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048838144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762193.923821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048838272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762193.923835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762193.923838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762193.923839:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762193.923841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1643 00000100:00080000:1.0:1789762193.923844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762193.923846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762193.923849:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762193.923856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048838400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762193.923863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762193.923866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762193.923867:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762193.923869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1643 00000100:00080000:1.0:1789762193.923871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762193.923874:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762193.923877:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762193.923883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048838528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762193.923889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762193.923891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762193.923896:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762193.923899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1643 00000100:00080000:1.0:1789762193.923902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762193.923904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762193.923908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762193.923916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048838656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762193.923962:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048838272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762193.924004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762193.924034:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762193.924078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762194.179528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762194.819613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762195.203570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762195.843606:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762196.227539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762196.867603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762196.995676:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762196.995743:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762196.995762:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762196.995804:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762196.995814:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762196.995824:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762196.995836:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789762196.995865:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762193/real 1789762196] req@ffff9811c92c5180 x1876700048838272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762196.995894:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048838272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762196.995907:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762196.995913:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762196.995918:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048838272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762196.995927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762193/real 1789762196] req@ffff9811c92c6d80 x1876700048838400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762196.995942:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048838400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762196.995951:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762196.995955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762196.995959:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762196.995966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762193/real 1789762196] req@ffff9811c92c5500 x1876700048838528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762196.995978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048838528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762196.995987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762196.995998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762196.996002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762196.996011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762193/real 1789762196] req@ffff9811c92c5880 x1876700048838656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762196.996027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048838656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762196.996036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762196.996040:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762196.996043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048838656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762196.996049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762193/real 1789762196] req@ffff9811c92c7b80 x1876700048838144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762196.996061:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048838144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762196.996068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762196.996071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762196.996074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048838144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762197.251579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762197.891659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762198.275566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762198.915638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762199.043526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762199.043535:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762199.043538:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762199.043547:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1648 00000100:00080000:1.0:1789762199.043556:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762199.043563:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762199.043594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048838784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762199.043619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762199.043622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762199.043623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762199.043626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1648 00000100:00080000:1.0:1789762199.043632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762199.043634:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762199.043641:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762199.043651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048838912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762199.043685:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048838784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762199.043693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762199.043697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762199.043699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762199.043703:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1648 00000100:00080000:1.0:1789762199.043707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762199.043712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762199.043718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762199.043732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048839040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762199.043742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762199.043746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789762199.043748:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762199.043750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762199.043757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1648 00000100:00080000:1.0:1789762199.043762:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762199.043767:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762199.043773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762199.043787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048839168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762199.043791:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762199/real 1789762199] req@ffff9811c92c5c00 x1876700048838784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762199.043814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762199.043819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762199.043822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762199.043825:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048838784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762199.043829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1648 00000100:00080000:1.0:1789762199.043844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762199.043851:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762199.043851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789762199.043861:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789762199.043862:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762199.043868:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048838784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762199.043881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048839296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762199.043899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048838912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762199.043927:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762199.043941:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762199/real 1789762199] req@ffff9811c92c6680 x1876700048838912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762199.043954:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048838912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762199.043962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762199.043966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762199.043970:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048838912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762199.043977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762199.044005:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762199.044016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762199/real 1789762199] req@ffff9811c92c4000 x1876700048839040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762199.044029:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048839040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762199.044038:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762199.044041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762199.044045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762199.044059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762199.044083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762199.044095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762199/real 1789762199] req@ffff9811c92c4a80 x1876700048839168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762199.044109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048839168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762199.044118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762199.044122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762199.044125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762199.044132:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762199.044153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762199.044164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762199/real 1789762199] req@ffff9811c92c4380 x1876700048839296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762199.044178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048839296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762199.044187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762199.044190:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762199.044194:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762199.299556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762199.939598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762200.323550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762200.963693:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762201.347525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762201.987586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762202.371523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762203.011628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762203.395515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762204.035563:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762204.163554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762204.163564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762204.163568:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762204.163577:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1653 00000100:00080000:1.0:1789762204.163586:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762204.163593:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762204.163645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048839424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762204.163664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762204.163667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762204.163669:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762204.163672:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1653 00000100:00080000:1.0:1789762204.163675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762204.163678:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762204.163685:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762204.163696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048839552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762204.163705:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048839424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762204.163708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762204.163726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762204.163728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762204.163732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1653 00000100:00080000:1.0:1789762204.163735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762204.163738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762204.163741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762204.163749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048839680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762204.163756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762204.163762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762204.163766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762204.163768:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1653 00000100:00080000:1.0:1789762204.163770:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762204.163772:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762204.163775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762204.163783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048839808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762204.163790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762204.163792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762204.163794:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762204.163797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1653 00000100:00080000:1.0:1789762204.163810:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762204.163812:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762204.163821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762204.163825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762204.163835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048839936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762204.163859:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762204/real 1789762204] req@ffff9811c92c5f80 x1876700048839424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762204.163875:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048839424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762204.163883:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762204.163885:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762204.163888:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048839424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762204.163908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048839552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762204.163922:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762204.163931:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762204/real 1789762204] req@ffff9811c92c7480 x1876700048839552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762204.163939:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048839552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762204.163945:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762204.163947:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762204.163949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048839552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762204.163953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762204.163963:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762204.163968:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762204/real 1789762204] req@ffff9811c92c4e00 x1876700048839680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762204.163990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048839680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762204.163996:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762204.163998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762204.164000:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762204.164004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762204.164040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762204.164053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762204/real 1789762204] req@ffff9811c92c6300 x1876700048839808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762204.164067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048839808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762204.164075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762204.164079:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762204.164081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762204.164086:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762204.164101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762204.164109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762204/real 1789762204] req@ffff9811c92c6a00 x1876700048839936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762204.164117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048839936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762204.164131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762204.164134:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762204.164137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048839936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762204.419533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762205.059683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762205.443522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762206.083714:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762206.467644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762207.107638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762207.491544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762208.131595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762208.515557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762209.155570:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762209.283587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762209.283613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762209.283619:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762209.283629:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1658 00000100:00080000:2.0:1789762209.283641:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762209.283650:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762209.283684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048840064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762209.283706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762209.283709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762209.283711:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762209.283713:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1658 00000100:00080000:2.0:1789762209.283716:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762209.283719:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762209.283726:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762209.283734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048840192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762209.283744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762209.283747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789762209.283748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048840064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762209.283765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762209.283772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1658 00000100:00080000:2.0:1789762209.283777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762209.283782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762209.283791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762209.283805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048840320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762209.283808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762209.283820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762209.283824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762209.283826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762209.283831:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1658 00000100:00080000:2.0:1789762209.283835:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762209.283839:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762209.283845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762209.283848:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762209/real 1789762209] req@ffff9811d8639880 x1876700048840064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762209.283861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048840448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762209.283868:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048840064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762209.283879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762209.283882:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762209.283884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762209.283888:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789762209.283890:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762209.283894:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048840064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762209.283899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1658 00000100:00080000:2.0:1789762209.283905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762209.283908:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762209.283913:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762209.283918:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048840192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762209.283942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048840576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762209.283946:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762209.283960:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762209/real 1789762209] req@ffff9811d8639500 x1876700048840192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762209.283970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048840192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762209.283977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762209.283979:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762209.283982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048840192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762209.283993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762209.284008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762209.284015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762209/real 1789762209] req@ffff9811d8639f80 x1876700048840320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762209.284024:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048840320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762209.284029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762209.284031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762209.284034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762209.284037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762209.284047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762209.284053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762209/real 1789762209] req@ffff9811d863a680 x1876700048840448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762209.284060:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048840448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762209.284065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762209.284067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762209.284069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762209.284078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762209.284088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762209.284094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762209/real 1789762209] req@ffff9811d8638a80 x1876700048840576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762209.284114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048840576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762209.284122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762209.284126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762209.284130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762209.539581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762210.179648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762210.563610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762211.203620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762211.587634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762212.227663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762212.611574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762213.251635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762213.635577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762214.275626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762214.403592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762214.403606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762214.403611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762214.403625:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1663 00000100:00080000:1.0:1789762214.403640:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762214.403649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762214.403696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048840704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762214.403725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762214.403728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762214.403729:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762214.403733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1663 00000100:00080000:1.0:1789762214.403735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762214.403738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762214.403748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762214.403757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048840832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762214.403767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762214.403769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762214.403771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762214.403774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1663 00000100:00100000:0.0:1789762214.403780:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048840704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762214.403797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762214.403804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762214.403810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762214.403826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048840960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762214.403838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762214.403842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762214.403843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762214.403848:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1663 00000800:00000100:0.0:1789762214.403853:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762214.403856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762214.403864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762214.403871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762214.403888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048841088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762214.403898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762214.403900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789762214.403903:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762214/real 1789762214] req@ffff9811c92c7100 x1876700048840704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762214.403904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762214.403912:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1663 00000100:00080000:1.0:1789762214.403920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762214.403927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762214.403931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048840704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762214.403935:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762214.403942:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762214.403946:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762214.403950:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048840704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762214.403955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048841216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762214.403962:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048840832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762214.403995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762214.404009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762214/real 1789762214] req@ffff9811c92c7800 x1876700048840832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762214.404018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048840832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762214.404035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762214.404038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762214.404040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048840832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762214.404062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762214.404078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762214.404085:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762214/real 1789762214] req@ffff9811c92c4700 x1876700048840960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762214.404093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048840960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762214.404098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762214.404100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762214.404102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048840960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762214.404105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762214.404116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762214.404121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762214/real 1789762214] req@ffff9811c92c6a00 x1876700048841088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762214.404129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048841088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762214.404133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762214.404135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762214.404142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762214.404146:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762214.404158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762214.404165:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762214/real 1789762214] req@ffff9811c92c6300 x1876700048841216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762214.404172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048841216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762214.404177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762214.404178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762214.404180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762214.659568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762215.299646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762215.683560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762216.323669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762216.707631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762217.347641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762217.731585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762218.371676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762218.755547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762219.395652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762219.523550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762219.523560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762219.523565:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762219.523576:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1669 00000100:00080000:0.0:1789762219.523589:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762219.523597:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762219.523639:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048841344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762219.523674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762219.523677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762219.523678:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762219.523682:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1669 00000100:00080000:0.0:1789762219.523684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762219.523699:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762219.523709:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762219.523719:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048841472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762219.523730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762219.523734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762219.523736:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762219.523739:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1669 00000100:00080000:0.0:1789762219.523741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762219.523744:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762219.523747:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762219.523754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048841600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762219.523763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762219.523765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762219.523768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762219.523773:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1669 00000100:00080000:0.0:1789762219.523777:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762219.523780:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048841344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762219.523782:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762219.523787:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762219.523804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048841728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762219.523811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762219.523818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762219.523820:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762219.523824:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1669 00000100:00080000:0.0:1789762219.523827:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762219.523830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762219.523835:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762219.523843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048841856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762219.523860:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762219.523902:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762219/real 1789762219] req@ffff9811c2e2ad80 x1876700048841344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762219.523918:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048841344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762219.523926:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762219.523930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762219.523933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048841344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762219.523939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048841472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762219.523955:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762219.523966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762219/real 1789762219] req@ffff9811c2e29880 x1876700048841472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762219.524002:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048841472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762219.524020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762219.524025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762219.524028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048841472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762219.524055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762219.524080:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762219.524093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762219/real 1789762219] req@ffff9811c2e2bb80 x1876700048841600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762219.524106:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048841600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762219.524116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762219.524120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762219.524124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762219.524131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762219.524148:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762219.524167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762219/real 1789762219] req@ffff9811c2e29180 x1876700048841728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762219.524183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048841728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762219.524204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762219.524208:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762219.524212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762219.524219:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762219.524240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762219.524248:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762219/real 1789762219] req@ffff9811c2e28e00 x1876700048841856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762219.524256:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048841856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762219.524262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762219.524264:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762219.524266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048841856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762219.779591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762220.419672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762220.803549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762221.443578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762221.827572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762222.467607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762222.851578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762223.491610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762223.875542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762224.515638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762224.643595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762224.643607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762224.643613:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762224.643624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1674 00000100:00080000:0.0:1789762224.643637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762224.643645:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762224.643683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048841984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762224.643713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762224.643718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762224.643720:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762224.643725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1674 00000100:00080000:0.0:1789762224.643731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762224.643737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762224.643748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762224.643756:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048841984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762224.643768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048842112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762224.643793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762224.643799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762224.643801:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762224.643808:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1674 00000100:00080000:0.0:1789762224.643814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762224.643815:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762224.643825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762224.643833:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762224.643849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048842240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762224.643854:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762224/real 1789762224] req@ffff9811c2e28700 x1876700048841984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762224.643875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762224.643881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789762224.643884:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048841984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762224.643886:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762224.643892:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1674 00000100:00080000:0.0:1789762224.643899:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762224.643906:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789762224.643907:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789762224.643913:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762224.643916:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762224.643924:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048841984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762224.643932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048842368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762224.643944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762224.643948:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789762224.643955:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048842112:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789762224.643960:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762224.643967:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1674 00000100:00080000:0.0:1789762224.643979:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762224.643984:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762224.643989:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762224.643996:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762224.644003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762224/real 1789762224] req@ffff9811c2e2a680 x1876700048842112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762224.644013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048842496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762224.644026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048842112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762224.644038:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762224.644043:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762224.644048:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048842112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762224.644057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762224.644083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762224.644096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762224/real 1789762224] req@ffff9811c2e29c00 x1876700048842240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762224.644107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048842240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762224.644114:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762224.644116:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762224.644119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762224.644139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762224.644157:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762224.644168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762224/real 1789762224] req@ffff9811c2e28e00 x1876700048842368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762224.644177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048842368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762224.644185:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762224.644188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762224.644192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762224.644207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762224.644224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762224.644239:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762224/real 1789762224] req@ffff9811c2e29180 x1876700048842496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762224.644250:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048842496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762224.644256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762224.644259:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762224.644261:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762224.899561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762225.539597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762225.923581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762226.563622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762226.947554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762227.587617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762227.971578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762228.611618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762228.995548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762229.635612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762229.763561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762229.763570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762229.763574:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762229.763583:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1679 00000100:00080000:0.0:1789762229.763594:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762229.763600:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762229.763632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048842624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762229.763653:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762229.763656:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762229.763657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762229.763660:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1679 00000100:00080000:0.0:1789762229.763662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762229.763665:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762229.763674:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762229.763689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048842752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762229.763693:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048842624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762229.763712:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762229.763732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762229.763734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762229.763747:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1679 00000800:00000100:2.0:1789762229.763751:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762229.763753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762229.763759:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762229.763764:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762229.763777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048842880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762229.763786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762229.763790:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762229/real 1789762229] req@ffff9811c2e2bb80 x1876700048842624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762229.763791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762229.763794:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762229.763803:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1679 00000100:00080000:0.0:1789762229.763807:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762229.763811:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762229.763815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762229.763819:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048842624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762229.763827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048843008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762229.763831:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762229.763838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762229.763842:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762229.763843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762229.763847:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762229.763852:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1679 00000100:00100000:2.0:1789762229.763853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048842624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762229.763855:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762229.763861:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762229.763870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762229.763879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048843136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762229.763886:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048842752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762229.763911:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762229.763921:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762229/real 1789762229] req@ffff9811c2e29880 x1876700048842752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762229.763931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048842752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762229.763937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762229.763940:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762229.763942:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048842752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762229.763946:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762229.763957:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762229.763964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762229/real 1789762229] req@ffff9811c2e2ad80 x1876700048842880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762229.763979:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048842880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762229.763985:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762229.763987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762229.763990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048842880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762229.763993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762229.764007:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762229.764015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762229/real 1789762229] req@ffff9811c2e2b100 x1876700048843008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762229.764027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048843008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762229.764036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762229.764040:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762229.764044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762229.764063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762229.764081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762229.764089:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762229/real 1789762229] req@ffff9811c2e2aa00 x1876700048843136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762229.764097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048843136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762229.764112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762229.764115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762229.764117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762230.019573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762230.659611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762231.043531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762231.683641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762232.067533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762232.707620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762233.091541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762233.731695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762234.115578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762234.755583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762234.883602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762234.883614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762234.883618:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762234.883629:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1684 00000100:00080000:2.0:1789762234.883641:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762234.883649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762234.883690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048843264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762234.883741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762234.883745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762234.883747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762234.883751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1684 00000100:00080000:2.0:1789762234.883754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762234.883759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762234.883768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762234.883799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048843392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762234.883813:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048843264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762234.883840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762234.883845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762234.883847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762234.883853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1684 00000100:00080000:2.0:1789762234.883858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762234.883862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762234.883869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762234.883899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048843520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762234.883909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762234.883917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789762234.883918:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762234.883922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762234.883932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1684 00000100:00080000:2.0:1789762234.883940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762234.883946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762234.883995:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762234.884002:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762234/real 1789762234] req@ffff9811d863aa00 x1876700048843264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762234.884035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048843648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762234.884048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048843264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762234.884063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762234.884067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762234.884070:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762234.884076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789762234.884078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762234.884084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048843264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762234.884092:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1684 00000100:00080000:2.0:1789762234.884110:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762234.884116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048843392:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789762234.884124:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762234.884137:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762234.884155:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762234.884163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048843776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762234.884177:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762234/real 1789762234] req@ffff9811d8638000 x1876700048843392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762234.884203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048843392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762234.884218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762234.884240:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762234.884247:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048843392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762234.884258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762234.884316:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762234.884330:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762234/real 1789762234] req@ffff9811d863b800 x1876700048843520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762234.884344:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048843520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762234.884352:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762234.884358:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762234.884363:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762234.884374:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762234.884445:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762234.884507:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762234/real 1789762234] req@ffff9811d863a300 x1876700048843648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762234.884523:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048843648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762234.884532:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762234.884536:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762234.884539:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762234.884559:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762234.884583:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762234.884595:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762234/real 1789762234] req@ffff9811d863b480 x1876700048843776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762234.884625:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048843776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762234.884634:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762234.884637:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762234.884640:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048843776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762235.139529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762235.779610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762236.163555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762236.803619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762237.187540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762237.827597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762238.211574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762238.851599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762239.235575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762239.875623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762240.003565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762240.003575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762240.003578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762240.003587:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1689 00000100:00080000:2.0:1789762240.003598:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762240.003605:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762240.003637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048843904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762240.003663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762240.003666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762240.003668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762240.003671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1689 00000100:00080000:2.0:1789762240.003674:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762240.003676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762240.003684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762240.003706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048844032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762240.003714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762240.003716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762240.003718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762240.003720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1689 00000100:00080000:2.0:1789762240.003723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762240.003726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762240.003729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762240.003736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048844160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762240.003744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762240.003746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762240.003748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762240.003753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048843904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762240.003753:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1689 00000100:00080000:2.0:1789762240.003758:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762240.003763:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762240.003767:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762240.003775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048844288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762240.003782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762240.003784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762240.003786:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762240.003793:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1689 00000100:00080000:2.0:1789762240.003797:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762240.003800:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762240.003804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762240.003814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048844416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762240.003818:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762240.003859:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762240/real 1789762240] req@ffff9811d8638380 x1876700048843904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762240.003879:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048843904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762240.003891:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762240.003895:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762240.003900:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048843904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762240.003908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048844032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762240.003931:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762240.003945:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762240/real 1789762240] req@ffff9811d8638e00 x1876700048844032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762240.003966:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048844032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762240.003972:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762240.003974:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762240.003977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048844032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762240.003995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762240.004011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762240.004019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762240/real 1789762240] req@ffff9811d8638700 x1876700048844160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762240.004027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048844160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762240.004033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762240.004035:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762240.004037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762240.004040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762240.004050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762240.004056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762240/real 1789762240] req@ffff9811d863b480 x1876700048844288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762240.004063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048844288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762240.004089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762240.004091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762240.004093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762240.004097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762240.004110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762240.004117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762240/real 1789762240] req@ffff9811d863a300 x1876700048844416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762240.004124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048844416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762240.004130:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762240.004132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762240.004134:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762240.259571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762240.899676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762241.283640:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762241.923664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762242.307537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762242.947653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762243.331561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762243.971600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762244.355588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762244.995636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762245.123573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762245.123582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762245.123586:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762245.123596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1694 00000100:00080000:2.0:1789762245.123607:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762245.123613:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762245.123647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048844544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762245.123671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762245.123674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762245.123676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762245.123679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1694 00000100:00080000:2.0:1789762245.123681:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762245.123683:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762245.123704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762245.123716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048844672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762245.123727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762245.123730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762245.123734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762245.123735:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048844544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762245.123739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1694 00000100:00080000:2.0:1789762245.123745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762245.123748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762245.123752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762245.123761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048844800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762245.123769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762245.123772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762245.123773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762245.123776:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1694 00000100:00080000:2.0:1789762245.123779:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762245.123781:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762245.123784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762245.123794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048844928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762245.123800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762245.123803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762245.123809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762245.123812:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762245.123816:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1694 00000100:00080000:2.0:1789762245.123821:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762245.123824:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762245.123829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762245.123837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048845056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762245.123868:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762245/real 1789762245] req@ffff9811d863b800 x1876700048844544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762245.123888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048844544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762245.123905:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762245.123909:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762245.123914:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048844544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762245.123969:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048844672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762245.124002:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762245.124015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762245/real 1789762245] req@ffff9811d8638000 x1876700048844672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762245.124026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048844672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762245.124033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762245.124052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762245.124054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048844672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762245.124060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762245.124074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762245.124081:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762245/real 1789762245] req@ffff9811d863aa00 x1876700048844800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762245.124089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048844800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762245.124095:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762245.124097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762245.124099:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762245.124109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762245.124124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762245.124132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762245/real 1789762245] req@ffff9811d8638a80 x1876700048844928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762245.124140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048844928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762245.124146:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762245.124148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762245.124150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048844928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762245.124154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762245.124165:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762245.124171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762245/real 1789762245] req@ffff9811d863a680 x1876700048845056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762245.124198:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048845056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762245.124204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762245.124206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762245.124208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762245.379604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762246.019654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762246.403602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762247.043616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762247.427555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762248.067641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762248.451561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762249.091592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762249.475533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762250.115645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762250.243565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762250.243580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762250.243585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762250.243599:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1699 00000100:00080000:3.0:1789762250.243612:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762250.243620:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762250.243663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048845184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762250.243689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762250.243692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762250.243693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762250.243696:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1699 00000100:00080000:3.0:1789762250.243699:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762250.243701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762250.243710:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762250.243719:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048845312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762250.243728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762250.243731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789762250.243748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048845184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762250.243751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762250.243755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1699 00000100:00080000:3.0:1789762250.243758:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762250.243767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762250.243774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762250.243791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048845440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762250.243803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762250.243807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762250.243810:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762250.243814:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1699 00000100:00080000:3.0:1789762250.243817:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762250.243820:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762250.243825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762250.243835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048845568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762250.243845:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762250.243846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762250.243852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762250.243854:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762250.243860:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1699 00000100:00080000:3.0:1789762250.243864:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762250.243869:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762250.243875:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762250.243894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048845696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762250.243899:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762250/real 1789762250] req@ffff9811e1a22680 x1876700048845184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762250.243919:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048845184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762250.243928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762250.243931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762250.243934:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048845184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762250.243940:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048845312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762250.243958:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762250.243966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762250/real 1789762250] req@ffff9811e1a22d80 x1876700048845312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762250.243974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048845312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762250.243980:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762250.243982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762250.243984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048845312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762250.244006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762250.244017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762250.244023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762250/real 1789762250] req@ffff9811e1a21880 x1876700048845440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762250.244042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048845440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762250.244047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762250.244049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762250.244051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762250.244055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762250.244068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762250.244074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762250/real 1789762250] req@ffff9811e1a21c00 x1876700048845568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762250.244081:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048845568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762250.244086:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762250.244088:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762250.244089:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762250.244093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762250.244102:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762250.244107:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762250/real 1789762250] req@ffff9811e1a22a00 x1876700048845696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762250.244113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048845696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762250.244127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762250.244130:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762250.244132:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048845696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762250.499557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762251.139620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762251.523581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762252.163637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762252.547530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762253.187607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762253.571537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762254.211642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762254.595544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762255.235627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762255.363617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762255.363627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762255.363632:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762255.363643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1704 00000100:00080000:2.0:1789762255.363655:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762255.363662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762255.363707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048845824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762255.363740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762255.363743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762255.363746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762255.363750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1704 00000100:00080000:2.0:1789762255.363753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762255.363756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762255.363765:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762255.363777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048845952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762255.363801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048845824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762255.363825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762255.363829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762255.363831:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762255.363837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1704 00000100:00080000:2.0:1789762255.363841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762255.363846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762255.363851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762255.363879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048846080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762255.363887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762255.363889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762255.363891:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762255.363893:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1704 00000100:00080000:2.0:1789762255.363895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762255.363900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762255.363901:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762255.363908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762255.363921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048846208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762255.363931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762255.363935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762255.363953:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762255.363961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1704 00000100:00100000:0.0:1789762255.363966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762255/real 1789762255] req@ffff9811d8639f80 x1876700048845824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762255.363984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762255.363997:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762255.364000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048845824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762255.364009:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789762255.364016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762255.364023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762255.364033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048845824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762255.364033:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048846336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762255.364074:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048845952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762255.364117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762255.364132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762255/real 1789762255] req@ffff9811d8639500 x1876700048845952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762255.364186:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048845952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762255.364208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762255.364237:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762255.364241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048845952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762255.364263:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762255.364288:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762255.364298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762255/real 1789762255] req@ffff9811d8639880 x1876700048846080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762255.364307:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048846080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762255.364313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762255.364315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762255.364317:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762255.364321:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762255.364333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762255.364339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762255/real 1789762255] req@ffff9811d863ad80 x1876700048846208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762255.364357:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048846208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762255.364363:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762255.364365:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762255.364366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762255.364370:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762255.364388:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762255.364394:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762255/real 1789762255] req@ffff9811d8639c00 x1876700048846336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762255.364402:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048846336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762255.364407:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762255.364408:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762255.364410:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762255.619613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762256.259603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762256.643609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762257.283611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762257.667553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762258.307683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762258.691575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762259.331671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762259.715558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762260.355620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762260.483583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762260.483592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762260.483597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762260.483608:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1710 00000100:00080000:1.0:1789762260.483620:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762260.483629:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762260.483678:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048846464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762260.483707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762260.483712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762260.483715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762260.483720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1710 00000100:00080000:1.0:1789762260.483726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762260.483741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762260.483753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762260.483755:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048846464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762260.483781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048846592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762260.483796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762260.483800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762260.483802:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762260.483821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1710 00000100:00080000:1.0:1789762260.483827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762260.483833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762260.483841:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762260.483841:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048846592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762260.483863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048846720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762260.483876:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762260.483880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762260.483883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762260.483887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846720:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762260.483890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1710 00000100:00080000:1.0:1789762260.483899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762260.483904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762260.483911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762260.483925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048846848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762260.483939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762260.483955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762260.483958:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762260.483963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846848:192.168.203.113@tcp:8:kworker.0 00000100:00080000:1.0:1789762260.483965:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1710 00000100:00080000:1.0:1789762260.483973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762260.483979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762260.483986:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762260.484001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048846976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762260.484052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762260.739556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762261.379598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762261.763644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762262.403695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762262.787596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762263.427603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762263.555733:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762263.555769:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762263.555783:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762263.555813:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762263.555819:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762263.555825:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762263.555830:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762263.555888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762260/real 1789762263] req@ffff9811c92c4a80 x1876700048846976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762263.555915:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048846976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762263.555928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762263.555934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762263.555939:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762263.555947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762260/real 1789762263] req@ffff9811c92c4380 x1876700048846848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762263.555984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048846848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762263.555994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762263.555998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762263.556002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762263.556009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762260/real 1789762263] req@ffff9811c92c5f80 x1876700048846720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762263.556023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048846720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762263.556030:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762263.556034:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762263.556038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048846720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762263.556045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762260/real 1789762263] req@ffff9811c92c7480 x1876700048846592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762263.556056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048846592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762263.556063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762263.556067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762263.556070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048846592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762263.556086:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762260/real 1789762263] req@ffff9811c92c4e00 x1876700048846464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762263.556108:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048846464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762263.556116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762263.556120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762263.556123:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048846464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762263.811523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762264.451611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762264.835562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762265.475590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762265.603573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762265.603582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762265.603585:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762265.603594:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1715 00000100:00080000:1.0:1789762265.603603:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762265.603609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762265.603650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048847104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762265.603681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762265.603684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762265.603686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762265.603700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1715 00000100:00080000:1.0:1789762265.603703:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762265.603706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762265.603713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762265.603724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048847232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762265.603733:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048847104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762265.603735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762265.603742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762265.603745:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762265.603749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1715 00000100:00080000:1.0:1789762265.603751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762265.603764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762265.603781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762265.603794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048847360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762265.603806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762265.603810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762265.603811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762265.603818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1715 00000800:00000100:2.0:1789762265.603819:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762265.603826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762265.603832:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762265.603839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762265.603853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048847488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762265.603862:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762265/real 1789762265] req@ffff9811c92c4000 x1876700048847104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762265.603868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762265.603878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762265.603881:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762265.603888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1715 00000100:00100000:2.0:1789762265.603889:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048847104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762265.603895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762265.603904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789762265.603905:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762265.603912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762265.603913:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762265.603922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048847104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762265.603932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048847616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762265.603952:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048847232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762265.603981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762265.603995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762265/real 1789762265] req@ffff9811c92c6680 x1876700048847232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762265.604011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048847232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762265.604035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762265.604050:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762265.604054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048847232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762265.604060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762265.604085:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762265.604098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762265/real 1789762265] req@ffff9811c92c5c00 x1876700048847360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762265.604109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048847360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762265.604117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762265.604121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762265.604124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762265.604129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762265.604168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762265.604207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762265/real 1789762265] req@ffff9811c92c7b80 x1876700048847488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762265.604222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048847488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762265.604231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762265.604235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762265.604245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762265.604252:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762265.604277:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762265.604290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762265/real 1789762265] req@ffff9811c92c5880 x1876700048847616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762265.604302:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048847616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762265.604310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762265.604313:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762265.604316:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048847616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762265.859545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762266.499604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762266.883515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762267.523574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762267.907502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762268.547591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762268.931503:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762269.571585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762269.955513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762270.595690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762270.723609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762270.723631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762270.723659:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762270.723688:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1720 00000100:00080000:1.0:1789762270.723701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762270.723709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762270.723754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048847744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762270.723801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762270.723804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762270.723806:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762270.723809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1720 00000100:00080000:1.0:1789762270.723812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762270.723815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762270.723824:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762270.723834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048847872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762270.723843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762270.723846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762270.723847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762270.723850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1720 00000100:00080000:1.0:1789762270.723852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762270.723854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762270.723857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762270.723864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048848000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762270.723881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762270.723884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762270.723886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762270.723889:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1720 00000100:00080000:1.0:1789762270.723892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762270.723897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762270.723900:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048847744:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762270.723902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762270.723916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048848128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762270.723924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762270.723926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762270.723928:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762270.723930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1720 00000100:00080000:1.0:1789762270.723933:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762270.723935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762270.723938:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762270.723946:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048848256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762270.723994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762270.724040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762270/real 1789762270] req@ffff9811c92c5500 x1876700048847744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762270.724058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048847744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762270.724067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762270.724071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762270.724083:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048847744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762270.724090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048847872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762270.724110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762270.724120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762270/real 1789762270] req@ffff9811c92c6d80 x1876700048847872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762270.724129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048847872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762270.724152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762270.724155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762270.724158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048847872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762270.724162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762270.724179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762270.724188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762270/real 1789762270] req@ffff9811c92c5180 x1876700048848000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762270.724200:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048848000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762270.724210:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762270.724213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762270.724217:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762270.724250:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762270.724271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762270.724288:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762270/real 1789762270] req@ffff9811c92c5880 x1876700048848128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762270.724297:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048848128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762270.724304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762270.724306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762270.724308:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762270.724312:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762270.724325:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762270.724331:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762270/real 1789762270] req@ffff9811c92c7b80 x1876700048848256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762270.724339:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048848256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762270.724345:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762270.724347:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762270.724349:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762270.979513:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762271.619586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762272.003507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762272.643587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762273.027553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762273.667593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762274.051526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762274.691686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762275.075560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762275.715593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762275.843633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762275.843643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762275.843666:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762275.843676:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1725 00000100:00080000:2.0:1789762275.843687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762275.843694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762275.843728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048848384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762275.843755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762275.843759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762275.843760:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762275.843764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1725 00000100:00080000:2.0:1789762275.843766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762275.843769:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762275.843777:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762275.843790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048848512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762275.843804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762275.843809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762275.843812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762275.843817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1725 00000100:00080000:2.0:1789762275.843823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762275.843829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762275.843837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762275.843838:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048848384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762275.843857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048848640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762275.843884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762275.843888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762275.843889:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762275.843894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1725 00000100:00080000:2.0:1789762275.843899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762275.843907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762275.843908:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762275.843914:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762275.843932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048848768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762275.843941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762275.843944:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762275.843946:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762275.843949:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1725 00000100:00080000:2.0:1789762275.843952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762275.843972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762275/real 1789762275] req@ffff9811d863b100 x1876700048848384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762275.843978:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762275.843990:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762275.844016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048848896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762275.844024:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048848384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762275.844051:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762275.844057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762275.844065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048848384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762275.844079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048848512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762275.844178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762275.844190:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762275/real 1789762275] req@ffff9811d863bb80 x1876700048848512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762275.844201:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048848512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762275.844208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762275.844210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762275.844213:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048848512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762275.844234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762275.844249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762275.844257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762275/real 1789762275] req@ffff9811d8639180 x1876700048848640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762275.844266:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048848640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762275.844272:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762275.844274:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762275.844277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762275.844281:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762275.844293:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762275.844300:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762275/real 1789762275] req@ffff9811d8639c00 x1876700048848768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762275.844323:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048848768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762275.844330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762275.844332:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762275.844334:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762275.844339:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762275.844353:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762275.844361:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762275/real 1789762275] req@ffff9811d863ad80 x1876700048848896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762275.844369:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048848896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762275.844375:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762275.844377:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762275.844379:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048848896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762276.099561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762276.739565:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762277.123520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762277.763524:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762278.147615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762278.787536:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762279.171549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762279.811556:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762280.195601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762280.835551:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762280.963570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762280.963609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762280.963613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762280.963626:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1730 00000100:00080000:1.0:1789762280.963683:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762280.963693:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762280.963738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048849024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762280.963769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762280.963774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762280.963776:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762280.963780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1730 00000100:00080000:1.0:1789762280.963785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762280.963790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762280.963803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762280.963816:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048849024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762280.963819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048849152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762280.963833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762280.963835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762280.963836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762280.963840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1730 00000100:00080000:1.0:1789762280.963843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762280.963846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762280.963860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762280.963870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048849280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762280.963877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762280.963886:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762280.963887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762280.963890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1730 00000100:00080000:1.0:1789762280.963894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762280.963898:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762280.963902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762280.963906:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762280.963915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048849408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762280.963923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762280.963926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762280.963927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762280.963930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1730 00000100:00080000:1.0:1789762280.963940:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762280.963946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762280.963949:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762280.963955:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762280/real 1789762280] req@ffff9811c92c5c00 x1876700048849024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762280.963960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048849536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762280.963987:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048849024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762280.964019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762280.964023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762280.964026:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048849024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762280.964050:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048849152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762280.964099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762280.964114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762280/real 1789762280] req@ffff9811c92c6680 x1876700048849152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762280.964129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048849152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762280.964138:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762280.964142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762280.964145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048849152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762280.964150:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762280.964166:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762280.964175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762280/real 1789762280] req@ffff9811c92c4000 x1876700048849280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762280.964183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048849280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762280.964188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762280.964191:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762280.964193:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762280.964202:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762280.964217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762280.964224:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762280/real 1789762280] req@ffff9811c92c4e00 x1876700048849408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762280.964231:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048849408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762280.964237:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762280.964239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762280.964241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762280.964244:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762280.964254:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762280.964260:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762280/real 1789762280] req@ffff9811c92c7480 x1876700048849536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762280.964267:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048849536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762280.964271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762280.964273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762280.964275:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762281.219612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762281.859578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762282.243563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762282.883592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762283.267502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762283.907597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762284.291570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762284.931626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762285.315636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762285.955667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762286.083679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762286.083704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762286.083709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762286.083719:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1735 00000100:00080000:3.0:1789762286.083731:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762286.083755:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762286.083790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048849664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762286.083818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762286.083822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762286.083825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762286.083829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1735 00000100:00080000:3.0:1789762286.083834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762286.083839:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762286.083849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762286.083864:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048849664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762286.083865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048849792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762286.083885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762286.083888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762286.083890:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762286.083894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1735 00000100:00080000:3.0:1789762286.083897:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762286.083901:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762286.083906:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762286.083917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048849920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762286.083927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762286.083930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762286.083939:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762286.083943:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1735 00000100:00080000:3.0:1789762286.083948:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762286.083952:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762286.083957:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789762286.083972:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762286.083973:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048850048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762286.083983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762286.083991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762286.083995:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762286.084002:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1735 00000100:00080000:3.0:1789762286.084030:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762286.084043:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762286.084052:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762286.084080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762286/real 1789762286] req@ffff9811e1a21180 x1876700048849664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762286.084087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048850176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762286.084148:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048849664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762286.084160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762286.084165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762286.084180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048849664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762286.084204:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048849792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762286.084230:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762286.084244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762286/real 1789762286] req@ffff9811e1a22300 x1876700048849792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762286.084263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048849792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762286.084275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762286.084308:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762286.084314:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048849792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762286.084324:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762286.084355:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762286.084384:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762286/real 1789762286] req@ffff9811e1a23100 x1876700048849920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762286.084396:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048849920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762286.084417:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762286.084438:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762286.084441:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048849920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762286.084448:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762286.084479:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762286.084489:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762286/real 1789762286] req@ffff9811e1a23800 x1876700048850048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762286.084501:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048850048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762286.084511:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762286.084515:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762286.084519:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762286.084560:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762286.084586:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762286.084602:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762286/real 1789762286] req@ffff9811e1a23480 x1876700048850176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762286.084621:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048850176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762286.084635:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762286.084640:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762286.084665:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762286.339567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762286.979678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762287.363590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762288.003713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762288.387572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762289.027609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762289.411574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762290.051634:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762290.435555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762291.075619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762291.203536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762291.203546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762291.203579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762291.203590:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1740 00000100:00080000:3.0:1789762291.203601:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762291.203613:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762291.203652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048850304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762291.203681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762291.203696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762291.203697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762291.203701:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1740 00000100:00080000:3.0:1789762291.203703:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762291.203705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762291.203714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762291.203725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048850432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762291.203734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762291.203737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762291.203739:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762291.203741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1740 00000100:00080000:3.0:1789762291.203743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762291.203746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762291.203760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762291.203767:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048850560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762291.203783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762291.203785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762291.203787:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762291.203790:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1740 00000100:00080000:3.0:1789762291.203792:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762291.203795:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762291.203799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762291.203808:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048850304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762291.203809:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048850688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762291.203832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762291.203834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762291.203836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762291.203839:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1740 00000100:00080000:3.0:1789762291.203842:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762291.203847:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762291.203850:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762291.203858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048850816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762291.203888:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762291.203928:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762291/real 1789762291] req@ffff9811e1a20000 x1876700048850304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762291.203944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048850304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762291.203952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762291.203956:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762291.203967:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048850304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762291.203972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048850432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762291.204004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762291.204014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762291/real 1789762291] req@ffff9811e1a20700 x1876700048850432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762291.204023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048850432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762291.204029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762291.204032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762291.204034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048850432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762291.204054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762291.204069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762291.204076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762291/real 1789762291] req@ffff9811e1a20380 x1876700048850560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762291.204084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048850560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762291.204090:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762291.204092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762291.204094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762291.204114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762291.204125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762291.204132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762291/real 1789762291] req@ffff9811e1a20e00 x1876700048850688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762291.204147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048850688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762291.204153:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762291.204155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762291.204157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762291.204161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762291.204172:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762291.204178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762291/real 1789762291] req@ffff9811e1a21500 x1876700048850816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762291.204196:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048850816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762291.204202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762291.204204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762291.204206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048850816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762291.459515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762292.099597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762292.483585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762293.123596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762293.507568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762294.147594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762294.531597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762295.171636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762295.555559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762296.195663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762296.323607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762296.323639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762296.323643:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762296.323654:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1745 00000100:00080000:3.0:1789762296.323664:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762296.323672:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762296.323710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048850944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762296.323748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762296.323752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762296.323754:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762296.323758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1745 00000100:00080000:3.0:1789762296.323760:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762296.323764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762296.323775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762296.323794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048851072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762296.323804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762296.323808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762296.323810:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762296.323812:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1745 00000100:00080000:3.0:1789762296.323816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762296.323819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762296.323823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762296.323834:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048850944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762296.323848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048851200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762296.323872:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762296.323876:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762296.323880:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762296.323887:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1745 00000100:00080000:3.0:1789762296.323891:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762296.323895:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762296.323902:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762296.323924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048851328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762296.323934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762296.323955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762296.323958:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762296.323961:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1745 00000100:00080000:3.0:1789762296.323966:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762296.323970:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762296.323980:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762296.323989:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762296.323998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048851456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762296.324081:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762296/real 1789762296] req@ffff9811e1a20a80 x1876700048850944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762296.324109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048850944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762296.324152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762296.324161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762296.324170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048850944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762296.324200:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048851072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762296.324254:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762296.324281:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762296/real 1789762296] req@ffff9811e1a21f80 x1876700048851072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762296.324301:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048851072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762296.324313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762296.324318:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762296.324322:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048851072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762296.324362:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762296.324391:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762296.324405:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762296/real 1789762296] req@ffff9811e1a21500 x1876700048851200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762296.324421:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048851200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762296.324431:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762296.324435:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762296.324441:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762296.324512:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762296.324546:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762296.324574:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762296/real 1789762296] req@ffff9811e1a20e00 x1876700048851328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762296.324604:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048851328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762296.324626:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762296.324637:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762296.324653:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762296.324674:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762296.324709:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762296.324725:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762296/real 1789762296] req@ffff9811e1a20380 x1876700048851456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762296.324740:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048851456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762296.324750:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762296.324754:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762296.324758:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762296.579594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762297.219576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762297.603565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762298.243581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762298.627548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762299.267629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762299.651599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762300.291590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762300.675533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762301.315594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762301.443566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762301.443576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762301.443579:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762301.443587:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1751 00000100:00080000:3.0:1789762301.443612:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762301.443619:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762301.443647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048851584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762301.443671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762301.443673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762301.443675:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762301.443678:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1751 00000100:00080000:3.0:1789762301.443680:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762301.443682:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762301.443689:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762301.443699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048851712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762301.443708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762301.443711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762301.443712:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762301.443716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1751 00000100:00080000:3.0:1789762301.443718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762301.443720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762301.443723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762301.443736:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048851840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762301.443740:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048851584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762301.443744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762301.443755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762301.443757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762301.443760:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1751 00000100:00080000:3.0:1789762301.443763:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762301.443766:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762301.443770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762301.443785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048851968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762301.443793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762301.443796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762301.443798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762301.443801:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1751 00000100:00080000:3.0:1789762301.443804:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762301.443807:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762301.443810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762301.443816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048852096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762301.443831:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762301.443879:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762301/real 1789762301] req@ffff9811e1a20700 x1876700048851584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762301.443924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048851584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762301.443935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762301.443940:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762301.443944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048851584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762301.443953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048851712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762301.443981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762301.443992:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762301/real 1789762301] req@ffff9811e1a20000 x1876700048851712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762301.444001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048851712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762301.444008:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762301.444010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762301.444013:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048851712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762301.444037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762301.444051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762301.444058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762301/real 1789762301] req@ffff9811e1a23480 x1876700048851840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762301.444066:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048851840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762301.444080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762301.444083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762301.444085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762301.444090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762301.444106:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762301.444112:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762301/real 1789762301] req@ffff9811e1a23800 x1876700048851968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762301.444120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048851968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762301.444126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762301.444128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762301.444129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048851968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762301.444133:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762301.444143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762301.444149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762301/real 1789762301] req@ffff9811e1a23100 x1876700048852096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762301.444156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048852096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762301.444161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762301.444163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762301.444171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762301.699616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762302.339637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762302.723541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762303.363603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762303.747600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762304.387605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762304.771536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762305.411606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762305.795581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762306.435596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762306.563583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762306.563594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762306.563598:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762306.563609:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1756 00000100:00080000:1.0:1789762306.563621:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762306.563628:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762306.563666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048852224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762306.563699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762306.563703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762306.563704:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762306.563708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1756 00000100:00080000:1.0:1789762306.563711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762306.563714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762306.563722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762306.563733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048852352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762306.563744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762306.563748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762306.563750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762306.563756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1756 00000100:00100000:0.0:1789762306.563759:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048852224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762306.563760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762306.563781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762306.563787:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762306.563800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048852480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762306.563811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762306.563814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762306.563816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762306.563820:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1756 00000100:00080000:1.0:1789762306.563824:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762306.563829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762306.563831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762306.563839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762306.563849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048852608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762306.563858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762306.563861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762306.563883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762306.563888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1756 00000100:00080000:1.0:1789762306.563892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762306.563897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762306.563902:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762306/real 1789762306] req@ffff9811c92c5f80 x1876700048852224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762306.563904:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762306.563922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048852736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762306.563931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048852224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762306.563946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762306.563950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762306.563956:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048852224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762306.563963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048852352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762306.564011:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762306.564024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762306/real 1789762306] req@ffff9811c92c4380 x1876700048852352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762306.564036:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048852352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762306.564044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762306.564047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762306.564050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048852352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762306.564080:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762306.564097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762306.564106:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762306/real 1789762306] req@ffff9811c92c4a80 x1876700048852480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762306.564116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048852480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762306.564124:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762306.564127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762306.564129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762306.564134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762306.564155:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762306.564163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762306/real 1789762306] req@ffff9811c92c6300 x1876700048852608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762306.564187:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048852608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762306.564194:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762306.564197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762306.564200:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762306.564205:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762306.564221:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762306.564229:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762306/real 1789762306] req@ffff9811c92c6a00 x1876700048852736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762306.564239:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048852736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762306.564246:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762306.564248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762306.564251:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048852736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762306.819551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762307.459581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762307.843594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762308.483621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762308.867571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762309.507707:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762309.891607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762310.531655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762310.915589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762311.555624:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762311.683608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762311.683620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762311.683625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762311.683637:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1761 00000100:00080000:2.0:1789762311.683648:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762311.683658:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762311.683697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048852864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762311.683751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762311.683755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762311.683757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762311.683762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1761 00000100:00080000:2.0:1789762311.683768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762311.683773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762311.683782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762311.683798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048852992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762311.683810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762311.683814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762311.683817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762311.683840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1761 00000100:00100000:0.0:1789762311.683842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048852864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762311.683845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762311.683853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762311.683859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762311.683875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048853120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762311.683887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762311.683900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762311.683903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762311.683906:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762311.683911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1761 00000100:00080000:2.0:1789762311.683925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762311.683931:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762311.683938:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762311.683947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762311/real 1789762311] req@ffff9811d8639880 x1876700048852864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762311.683956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048853248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762311.683982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762311.683988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762311.683990:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762311.683995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048852864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762311.683999:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1761 00000100:00080000:2.0:1789762311.684008:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762311.684012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762311.684017:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789762311.684020:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762311.684025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048852864:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789762311.684026:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762311.684033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048852992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762311.684044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048853376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762311.684055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762311.684068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762311/real 1789762311] req@ffff9811d8639500 x1876700048852992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762311.684079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048852992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762311.684085:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762311.684088:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762311.684090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048852992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762311.684109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762311.684120:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762311.684126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762311/real 1789762311] req@ffff9811d8639f80 x1876700048853120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762311.684133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048853120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762311.684148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762311.684151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762311.684153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762311.684157:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762311.684172:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762311.684178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762311/real 1789762311] req@ffff9811d863a680 x1876700048853248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762311.684186:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048853248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762311.684191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762311.684193:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762311.684195:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762311.684198:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762311.684208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762311.684214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762311/real 1789762311] req@ffff9811d8638a80 x1876700048853376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762311.684221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048853376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762311.684226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762311.684228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762311.684235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762311.939594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762312.579663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762312.963574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762313.603791:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762313.987561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762314.627713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762315.011540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762315.651654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762316.035574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762316.675610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762316.803584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762316.803618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762316.803622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762316.803631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1766 00000100:00080000:1.0:1789762316.803640:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762316.803645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762316.803675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048853504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762316.803697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762316.803700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762316.803702:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762316.803705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1766 00000100:00080000:1.0:1789762316.803707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762316.803709:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762316.803716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762316.803724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048853632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762316.803732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762316.803735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762316.803736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762316.803739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1766 00000100:00080000:1.0:1789762316.803741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762316.803743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762316.803746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762316.803751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048853504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762316.803756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048853760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762316.803769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762316.803772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762316.803774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762316.803777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1766 00000100:00080000:1.0:1789762316.803779:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762316.803782:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762316.803785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762316.803806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048853888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762316.803812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762316.803814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762316.803816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762316.803819:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1766 00000100:00080000:1.0:1789762316.803821:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762316.803824:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789762316.803828:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762316.803829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762316.803839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048854016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762316.803868:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762316/real 1789762316] req@ffff9811c92c4700 x1876700048853504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762316.803883:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048853504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762316.803892:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762316.803895:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762316.803898:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048853504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762316.803903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048853632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762316.803917:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762316.803924:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762316/real 1789762316] req@ffff9811c92c7800 x1876700048853632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762316.803932:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048853632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762316.803937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762316.803939:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762316.803941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048853632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762316.803969:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762316.803983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762316.803990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762316/real 1789762316] req@ffff9811c92c7100 x1876700048853760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762316.803998:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048853760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762316.804003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762316.804005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762316.804007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762316.804011:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762316.804020:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762316.804026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762316/real 1789762316] req@ffff9811c92c6a00 x1876700048853888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762316.804033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048853888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762316.804037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762316.804039:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762316.804041:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048853888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762316.804044:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762316.804053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762316.804058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762316/real 1789762316] req@ffff9811c92c6300 x1876700048854016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762316.804071:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048854016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762316.804076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762316.804078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762316.804080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762317.059580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762317.699605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762318.083528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762318.723601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762319.107560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762319.747643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762320.131538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762320.771676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762321.155619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762321.795627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762321.923589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762321.923599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762321.923603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762321.923615:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1771 00000100:00080000:3.0:1789762321.923627:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762321.923635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762321.923678:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048854144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762321.923702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762321.923705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762321.923706:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762321.923709:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1771 00000100:00080000:3.0:1789762321.923711:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762321.923714:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762321.923723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762321.923732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048854272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762321.923742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762321.923745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762321.923746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762321.923764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048854144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762321.923764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1771 00000100:00080000:3.0:1789762321.923772:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762321.923779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762321.923805:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762321.923816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048854400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762321.923824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762321.923827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762321.923828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762321.923831:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1771 00000100:00080000:3.0:1789762321.923835:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762321.923838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762321.923842:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762321.923851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048854528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762321.923853:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762321.923861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762321.923864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762321.923866:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762321.923868:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1771 00000100:00080000:3.0:1789762321.923870:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762321.923873:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762321.923877:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762321.923883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048854656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762321.923914:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762321/real 1789762321] req@ffff9811e1a22300 x1876700048854144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762321.923938:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048854144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762321.923952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762321.923956:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762321.923960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048854144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762321.923989:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048854272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762321.924015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762321.924029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762321/real 1789762321] req@ffff9811e1a21180 x1876700048854272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762321.924053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048854272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762321.924062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762321.924066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762321.924070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048854272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762321.924076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762321.924097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762321.924108:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762321/real 1789762321] req@ffff9811e1a22a00 x1876700048854400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762321.924121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048854400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762321.924129:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762321.924132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762321.924136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762321.924150:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762321.924168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762321.924177:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762321/real 1789762321] req@ffff9811e1a21c00 x1876700048854528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762321.924188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048854528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762321.924209:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762321.924213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762321.924217:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762321.924223:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762321.924241:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762321.924251:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762321/real 1789762321] req@ffff9811e1a21880 x1876700048854656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762321.924259:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048854656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762321.924264:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762321.924266:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762321.924268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048854656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762322.179574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762322.819600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762323.203611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762323.843589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762324.227632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762324.867549:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762325.251574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762325.891663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762326.275548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762326.915548:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762327.043563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762327.043571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762327.043574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762327.043583:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1776 00000100:00080000:2.0:1789762327.043605:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762327.043610:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762327.043637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048854784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762327.043658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762327.043661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762327.043663:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762327.043666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1776 00000100:00080000:2.0:1789762327.043668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762327.043670:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762327.043676:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762327.043685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048854912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762327.043703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762327.043706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762327.043707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762327.043721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1776 00000100:00080000:2.0:1789762327.043725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762327.043728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048854784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762327.043730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762327.043739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762327.043749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048855040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762327.043757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762327.043765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762327.043767:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762327.043770:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1776 00000100:00080000:2.0:1789762327.043774:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762327.043777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762327.043781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762327.043790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048855168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762327.043797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762327.043799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762327.043801:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762327.043803:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1776 00000100:00080000:2.0:1789762327.043806:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762327.043808:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762327.043812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762327.043818:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048854912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762327.043822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048855296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762327.043856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762327.043884:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762327.043897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762327.299559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762327.939580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762328.323541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762328.963553:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762329.347527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762329.987576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762330.115703:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762330.115757:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762330.115782:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762330.115821:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762330.115831:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762330.115840:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762330.115851:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789762330.115877:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762327/real 1789762330] req@ffff9811d863b800 x1876700048855040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762330.115905:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048855040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762330.115918:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762330.115924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762330.115929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762330.115938:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762327/real 1789762330] req@ffff9811d863a300 x1876700048855168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762330.115953:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048855168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762330.115962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762330.115966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762330.115970:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762330.115978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762327/real 1789762330] req@ffff9811d863b480 x1876700048855296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762330.115992:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048855296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762330.116002:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762330.116006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762330.116010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762330.116035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762327/real 1789762330] req@ffff9811d863aa00 x1876700048854784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762330.116052:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048854784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762330.116062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762330.116072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762330.116077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048854784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762330.116084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762327/real 1789762330] req@ffff9811d8638000 x1876700048854912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762330.116098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048854912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762330.116108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762330.116111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762330.116115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048854912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762330.371512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762331.011574:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762331.395566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762332.035588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762332.163623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762332.163631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762332.163635:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762332.163645:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1781 00000100:00080000:1.0:1789762332.163684:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762332.163694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762332.163745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048855424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762332.163801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762332.163807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762332.163809:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762332.163815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1781 00000100:00080000:1.0:1789762332.163819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762332.163825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762332.163851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762332.163863:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048855424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762332.163874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048855552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762332.163888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762332.163891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762332.163892:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762332.163895:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1781 00000100:00080000:1.0:1789762332.163900:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762332.163904:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762332.163909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762332.163922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048855680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762332.163932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762332.163944:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762332.163946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762332.163950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762332.163956:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1781 00000100:00080000:1.0:1789762332.163962:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762332.163968:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762332.163974:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762332.163994:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762332/real 1789762332] req@ffff9811c92c4a80 x1876700048855424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762332.164009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048855808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762332.164037:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762332.164042:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762332.164044:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762332.164049:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1781 00000100:00080000:1.0:1789762332.164053:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762332.164055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048855424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762332.164060:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762332.164072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762332.164076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762332.164084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762332.164090:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048855424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762332.164095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048855936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762332.164120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048855552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762332.164150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762332.164164:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762332/real 1789762332] req@ffff9811c92c4380 x1876700048855552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762332.164209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048855552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762332.164219:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762332.164222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762332.164226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048855552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762332.164232:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762332.164254:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762332.164267:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762332/real 1789762332] req@ffff9811c92c5f80 x1876700048855680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762332.164280:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048855680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762332.164303:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762332.164306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762332.164310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762332.164316:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762332.164342:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762332.164353:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762332/real 1789762332] req@ffff9811c92c7480 x1876700048855808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762332.164365:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048855808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762332.164373:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762332.164376:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762332.164380:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762332.164386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762332.164406:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762332.164417:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762332/real 1789762332] req@ffff9811c92c4e00 x1876700048855936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762332.164428:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048855936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762332.164434:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762332.164438:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762332.164448:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048855936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762332.419515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762333.059740:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762333.443545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762334.083586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762334.467515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762335.107630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762335.491508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762336.131673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762336.515507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762337.155592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762337.283549:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762337.283558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762337.283561:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762337.283569:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1786 00000100:00080000:1.0:1789762337.283578:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762337.283583:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762337.283615:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048856064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762337.283664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762337.283684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762337.283686:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762337.283691:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1786 00000100:00080000:1.0:1789762337.283695:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762337.283698:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762337.283708:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762337.283721:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048856192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762337.283733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762337.283736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762337.283738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762337.283741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1786 00000100:00080000:1.0:1789762337.283743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762337.283746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762337.283750:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762337.283762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048856320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762337.283764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048856064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762337.283783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762337.283787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762337.283788:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762337.283791:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1786 00000100:00080000:1.0:1789762337.283802:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762337.283806:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762337.283812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762337.283825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048856448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762337.283835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762337.283838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762337.283841:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762337.283843:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1786 00000100:00080000:1.0:1789762337.283847:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762337.283849:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762337.283854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762337.283856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762337.283866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048856576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762337.283915:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762337/real 1789762337] req@ffff9811c92c4000 x1876700048856064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762337.283941:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048856064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762337.283952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762337.283955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762337.283960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048856064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762337.283983:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048856192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762337.284002:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762337.284011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762337/real 1789762337] req@ffff9811c92c6680 x1876700048856192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762337.284020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048856192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762337.284025:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762337.284028:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762337.284047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048856192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762337.284093:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762337.284117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762337.284133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762337/real 1789762337] req@ffff9811c92c5c00 x1876700048856320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762337.284151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048856320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762337.284163:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762337.284168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762337.284173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762337.284182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762337.284209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762337.284224:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762337/real 1789762337] req@ffff9811c92c7b80 x1876700048856448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762337.284253:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048856448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762337.284262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762337.284266:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762337.284270:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762337.284277:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762337.284294:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762337.284302:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762337/real 1789762337] req@ffff9811c92c5880 x1876700048856576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762337.284319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048856576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762337.284327:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762337.284331:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762337.284335:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762337.539501:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762338.179692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762338.563505:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762339.203590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762339.587661:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762340.227602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762340.611591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762341.251643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762341.635554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762342.275639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762342.403660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762342.403669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762342.403673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762342.403681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1791 00000100:00080000:2.0:1789762342.403691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762342.403697:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762342.403742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048856704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762342.403771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762342.403774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762342.403775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762342.403778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1791 00000100:00080000:2.0:1789762342.403780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762342.403782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762342.403789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762342.403816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048856832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762342.403828:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048856704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762342.403832:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762342.403845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762342.403847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762342.403851:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1791 00000100:00080000:2.0:1789762342.403854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762342.403857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762342.403862:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762342.403872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048856960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762342.403880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762342.403883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762342.403884:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762342.403888:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1791 00000100:00080000:2.0:1789762342.403890:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762342.403892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762342.403895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762342.403903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048857088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762342.403909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762342.403911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762342.403914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762342.403925:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762342.403930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1791 00000100:00080000:2.0:1789762342.403935:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762342.403939:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762342.403944:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762342.403955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048857216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762342.403984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762342/real 1789762342] req@ffff9811d8638700 x1876700048856704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762342.404005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048856704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762342.404032:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762342.404036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762342.404039:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048856704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762342.404066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048856832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762342.404087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762342.404113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762342/real 1789762342] req@ffff9811d8638e00 x1876700048856832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762342.404121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048856832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762342.404126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762342.404128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762342.404131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048856832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762342.404134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762342.404146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762342.404152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762342/real 1789762342] req@ffff9811d8638380 x1876700048856960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762342.404175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048856960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762342.404181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762342.404183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762342.404185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048856960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762342.404195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762342.404211:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762342.404218:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762342/real 1789762342] req@ffff9811d8638000 x1876700048857088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762342.404227:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048857088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762342.404232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762342.404234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762342.404236:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762342.404240:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762342.404252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762342.404258:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762342/real 1789762342] req@ffff9811d863aa00 x1876700048857216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762342.404266:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048857216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762342.404271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762342.404273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762342.404275:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762342.659564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762343.299614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762343.683559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762344.323620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762344.707609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762345.347677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762345.731647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762346.371708:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762346.755617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762347.395666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762347.523575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762347.523589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762347.523594:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762347.523609:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1797 00000100:00080000:2.0:1789762347.523627:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762347.523639:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762347.523706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048857344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762347.523744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762347.523748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762347.523750:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762347.523754:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1797 00000100:00080000:2.0:1789762347.523758:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762347.523762:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762347.523774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762347.523817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048857472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762347.523825:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048857344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762347.523838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762347.523860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762347.523865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762347.523872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1797 00000100:00080000:2.0:1789762347.523876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762347.523882:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762347.523903:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762347.523926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048857600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762347.523943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762347.523949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762347.523952:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762347.523958:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1797 00000800:00000100:0.0:1789762347.523967:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762347.523969:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762347.523979:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762347.523988:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762347.524006:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048857728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762347.524017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762347.524021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762347.524023:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762347.524032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762347/real 1789762347] req@ffff9811d863b480 x1876700048857344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762347.524038:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1797 00000100:00080000:2.0:1789762347.524044:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762347.524049:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762347.524055:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762347.524083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048857856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762347.524188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048857344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762347.524210:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762347.524238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762347.524246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048857344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762347.524289:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048857472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762347.524337:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762347.524358:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762347/real 1789762347] req@ffff9811d863a300 x1876700048857472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762347.524375:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048857472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762347.524387:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762347.524391:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762347.524396:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048857472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762347.524404:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762347.524426:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762347.524439:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762347/real 1789762347] req@ffff9811d863b800 x1876700048857600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762347.524454:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048857600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762347.524483:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762347.524488:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762347.524492:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762347.524608:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762347.524643:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762347.524660:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762347/real 1789762347] req@ffff9811d8638a80 x1876700048857728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762347.524676:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048857728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762347.524704:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762347.524708:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762347.524712:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762347.524719:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762347.524739:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762347.524750:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762347/real 1789762347] req@ffff9811d863a680 x1876700048857856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762347.524762:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048857856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762347.524771:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762347.524774:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762347.524789:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048857856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762347.779664:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762348.419709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762348.803635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762349.443728:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762349.827569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762350.467613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762350.851594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762351.491615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762351.875560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762352.515626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762352.643538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762352.643550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762352.643554:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762352.643565:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1802 00000100:00080000:2.0:1789762352.643576:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762352.643583:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762352.643616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048857984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762352.643638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762352.643641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762352.643642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762352.643646:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1802 00000100:00080000:2.0:1789762352.643650:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762352.643654:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762352.643664:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762352.643680:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048857984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762352.643681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048858112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762352.643705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762352.643708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762352.643710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762352.643714:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1802 00000100:00080000:2.0:1789762352.643718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762352.643721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762352.643726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762352.643735:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762352.643739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048858240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762352.643748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762352.643751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762352.643753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762352.643756:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1802 00000100:00080000:2.0:1789762352.643758:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762352.643760:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762352.643763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762352.643771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048858368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762352.643779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762352.643781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762352.643782:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762352.643786:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1802 00000100:00100000:0.0:1789762352.643789:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762352/real 1789762352] req@ffff9811d8639f80 x1876700048857984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762352.643795:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762352.643803:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762352.643811:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762352.643822:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048857984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762352.643823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048858496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762352.643853:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762352.643860:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762352.643867:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048857984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762352.643892:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048858112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762352.643919:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762352.643934:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762352/real 1789762352] req@ffff9811d8639500 x1876700048858112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762352.643949:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048858112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762352.643959:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762352.643963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762352.643966:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048858112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762352.643970:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789762352.643989:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762352.644011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762352/real 1789762352] req@ffff9811d8639880 x1876700048858240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762352.644025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048858240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762352.644035:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762352.644037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762352.644040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762352.644044:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762352.644061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762352.644069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762352/real 1789762352] req@ffff9811d863ad80 x1876700048858368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762352.644077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048858368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762352.644082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762352.644085:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762352.644087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789762352.644090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789762352.644099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762352.644105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762352/real 1789762352] req@ffff9811d8639c00 x1876700048858496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762352.644112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048858496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762352.644122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762352.644124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789762352.644127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762352.899576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762353.539651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762353.923542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762354.563630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762354.947550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762355.587594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762355.971549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762356.611655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762356.995544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762357.635642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762357.763641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762357.763652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762357.763657:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762357.763668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1807 00000100:00080000:0.0:1789762357.763680:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762357.763688:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762357.763730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048858624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762357.763762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762357.763767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762357.763769:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762357.763772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1807 00000100:00080000:0.0:1789762357.763776:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762357.763780:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762357.763789:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762357.763802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048858752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762357.763812:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048858624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762357.763818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762357.763824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762357.763845:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762357.763851:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1807 00000100:00080000:0.0:1789762357.763856:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762357.763861:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762357.763867:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762357.763880:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048858880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762357.763891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762357.763897:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762357.763898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762357.763903:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762357.763910:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1807 00000100:00080000:0.0:1789762357.763915:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762357.763919:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762357.763924:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762357.763936:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048859008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762357.763946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762357.763949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762357.763967:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762357.763972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1807 00000100:00080000:0.0:1789762357.763976:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762357.763982:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762357.763985:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762357/real 1789762357] req@ffff9811c2e28380 x1876700048858624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762357.763987:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762357.764009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048859136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762357.764018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048858624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762357.764033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762357.764037:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762357.764043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048858624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762357.764069:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048858752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762357.764094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762357.764123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762357/real 1789762357] req@ffff9811c2e28a80 x1876700048858752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762357.764135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048858752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762357.764160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762357.764164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762357.764167:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048858752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762357.764173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762357.764194:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762357.764203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762357/real 1789762357] req@ffff9811c2e29f80 x1876700048858880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762357.764215:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048858880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762357.764222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762357.764225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762357.764228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048858880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762357.764233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762357.764249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762357.764257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762357/real 1789762357] req@ffff9811c2e2b800 x1876700048859008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762357.764267:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048859008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762357.764273:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762357.764276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762357.764286:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762357.764292:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762357.764311:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762357.764328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762357/real 1789762357] req@ffff9811c2e2b480 x1876700048859136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762357.764360:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048859136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762357.764367:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762357.764370:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762357.764372:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762358.019583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762358.659614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762359.043611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762359.683677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762360.067594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762360.707616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762361.091535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762361.731563:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762362.115548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762362.755578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762362.883552:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762362.883561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762362.883565:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762362.883577:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1812 00000100:00080000:1.0:1789762362.883586:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762362.883592:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762362.883624:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048859264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762362.883645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762362.883663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762362.883665:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762362.883668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1812 00000100:00080000:1.0:1789762362.883672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762362.883675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762362.883682:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048859264:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762362.883683:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762362.883709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048859392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762362.883719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762362.883722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762362.883724:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762362.883727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1812 00000100:00080000:1.0:1789762362.883729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762362.883732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762362.883735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762362.883741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048859520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762362.883748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762362.883751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762362.883752:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762362.883755:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1812 00000100:00080000:1.0:1789762362.883757:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762362.883772:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762362.883773:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762362.883784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762362.883794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048859648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762362.883802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762362.883804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762362.883805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762362.883808:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1812 00000100:00080000:1.0:1789762362.883811:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762362.883814:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762362.883818:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762362/real 1789762362] req@ffff9811c92c5180 x1876700048859264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762362.883820:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762362.883835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048859776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762362.883840:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048859264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762362.883853:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762362.883857:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762362.883861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048859264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762362.883878:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048859392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762362.883904:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762362.883913:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762362/real 1789762362] req@ffff9811c92c6d80 x1876700048859392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762362.883922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048859392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762362.883928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762362.883931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762362.883933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048859392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762362.883937:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762362.883948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762362.883953:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762362/real 1789762362] req@ffff9811c92c5500 x1876700048859520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762362.883961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048859520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762362.883966:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762362.883968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762362.883970:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762362.883983:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762362.883999:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762362.884006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762362/real 1789762362] req@ffff9811c92c5880 x1876700048859648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762362.884025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048859648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762362.884031:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762362.884033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762362.884035:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762362.884038:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762362.884056:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762362.884062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762362/real 1789762362] req@ffff9811c92c7b80 x1876700048859776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762362.884070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048859776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762362.884075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762362.884077:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762362.884079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048859776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762363.139537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762363.779577:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762364.163560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762364.803587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762365.187545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762365.827567:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762366.211561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762366.851625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762367.235611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762367.875582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762368.003595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762368.003607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762368.003611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762368.003620:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1817 00000100:00080000:2.0:1789762368.003630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762368.003649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762368.003680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048859904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762368.003702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762368.003705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762368.003707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762368.003710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1817 00000100:00080000:2.0:1789762368.003714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762368.003717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762368.003724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762368.003739:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048859904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762368.003739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048860032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762368.003756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762368.003759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762368.003761:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762368.003765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1817 00000100:00080000:2.0:1789762368.003768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762368.003772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762368.003775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762368.003784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048860160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762368.003789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762368.003793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762368.003799:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762368.003803:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762368.003807:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1817 00000100:00080000:2.0:1789762368.003813:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762368.003817:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762368.003821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762368.003831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048860288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762368.003837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762368.003840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762368.003841:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762368.003847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1817 00000100:00100000:1.0:1789762368.003848:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762368/real 1789762368] req@ffff9811d8639180 x1876700048859904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762368.003850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762368.003857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762368.003864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762368.003874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048860416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762368.003883:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048859904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762368.003894:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762368.003898:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762368.003902:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048859904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762368.003923:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048860032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762368.003952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762368.003965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762368/real 1789762368] req@ffff9811d863bb80 x1876700048860032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762368.003978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048860032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762368.003985:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762368.003988:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762368.003990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048860032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762368.003995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762368.004009:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762368.004018:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762368/real 1789762368] req@ffff9811d863b100 x1876700048860160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762368.004027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048860160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762368.004033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762368.004036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762368.004039:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762368.004044:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762368.004057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762368.004065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762368/real 1789762368] req@ffff9811d8639c00 x1876700048860288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762368.004090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048860288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762368.004096:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762368.004099:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762368.004101:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762368.004105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762368.004116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762368.004130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762368/real 1789762368] req@ffff9811d863ad80 x1876700048860416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762368.004137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048860416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762368.004143:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762368.004145:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762368.004147:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762368.259531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762368.899599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762369.283620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762369.923549:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762370.307544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762370.947623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762371.331562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762371.971632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762372.355543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762372.995668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762373.123597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762373.123610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762373.123616:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762373.123626:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1822 00000100:00080000:1.0:1789762373.123636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762373.123644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762373.123683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048860544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762373.123729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762373.123732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762373.123733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762373.123737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1822 00000100:00080000:1.0:1789762373.123741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762373.123745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762373.123753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762373.123767:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048860544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762373.123768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048860672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762373.123785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762373.123788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762373.123789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762373.123792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1822 00000100:00080000:1.0:1789762373.123794:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762373.123797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762373.123800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762373.123807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048860800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762373.123814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762373.123816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762373.123817:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762373.123820:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1822 00000100:00080000:1.0:1789762373.123847:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762373.123850:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762373.123856:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762373.123856:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762373.123869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048860928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762373.123877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762373.123880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762373.123881:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762373.123884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1822 00000100:00080000:1.0:1789762373.123887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762373.123889:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762373.123893:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762373.123899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048861056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762373.123916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762373/real 1789762373] req@ffff9811c92c5c00 x1876700048860544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762373.123937:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048860544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762373.123958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762373.123963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762373.123967:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048860544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762373.123989:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048860672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762373.124008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762373.124017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762373/real 1789762373] req@ffff9811c92c6680 x1876700048860672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762373.124026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048860672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762373.124032:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762373.124034:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762373.124037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048860672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762373.124040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762373.124050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762373.124066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762373/real 1789762373] req@ffff9811c92c4000 x1876700048860800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762373.124073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048860800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762373.124079:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762373.124081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762373.124083:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762373.124092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762373.124107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762373.124113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762373/real 1789762373] req@ffff9811c92c4e00 x1876700048860928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762373.124121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048860928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762373.124127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762373.124128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762373.124130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048860928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762373.124144:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762373.124154:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762373.124159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762373/real 1789762373] req@ffff9811c92c7480 x1876700048861056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762373.124167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048861056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762373.124171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762373.124173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762373.124175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762373.379562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762374.019669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762374.403544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762375.043589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762375.427535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762376.067609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762376.451588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762377.091669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762377.475619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762378.115610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762378.243559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762378.243586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762378.243589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762378.243599:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1827 00000100:00080000:2.0:1789762378.243610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762378.243616:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762378.243648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048861184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762378.243691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762378.243694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762378.243696:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762378.243699:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1827 00000100:00080000:2.0:1789762378.243702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762378.243705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762378.243712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762378.243723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048861312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762378.243736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762378.243741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762378.243743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762378.243747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1827 00000100:00080000:2.0:1789762378.243751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762378.243767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762378.243774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762378.243788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048861440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762378.243798:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048861184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762378.243817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762378.243821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762378.243834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762378.243838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1827 00000100:00080000:2.0:1789762378.243842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762378.243846:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762378.243851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762378.243862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048861568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762378.243869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762378.243872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762378.243874:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762378.243876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1827 00000100:00080000:2.0:1789762378.243879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762378.243881:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762378.243884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762378.243890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762378.243894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048861696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762378.243948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762378/real 1789762378] req@ffff9811d8639880 x1876700048861184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762378.243965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048861184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762378.243975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762378.243979:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762378.243983:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048861184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762378.243999:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048861312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762378.244020:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762378.244031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762378/real 1789762378] req@ffff9811d8639500 x1876700048861312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762378.244040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048861312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762378.244047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762378.244049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762378.244052:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048861312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762378.244075:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762378.244099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762378.244123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762378/real 1789762378] req@ffff9811d8639f80 x1876700048861440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762378.244137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048861440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762378.244148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762378.244152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762378.244156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762378.244163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762378.244187:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762378.244201:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762378/real 1789762378] req@ffff9811d863a680 x1876700048861568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762378.244227:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048861568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762378.244238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762378.244241:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762378.244246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762378.244253:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762378.244277:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762378.244290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762378/real 1789762378] req@ffff9811d8638a80 x1876700048861696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762378.244306:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048861696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762378.244316:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762378.244320:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762378.244324:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048861696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762378.499522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762379.139593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762379.523530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762380.163582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762380.547526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762381.187578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762381.571574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762382.211562:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762382.595520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762383.235555:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762383.363596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762383.363606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762383.363609:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762383.363618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1832 00000100:00080000:3.0:1789762383.363629:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762383.363635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762383.363670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048861824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762383.363739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762383.363742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762383.363744:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762383.363748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1832 00000100:00080000:3.0:1789762383.363752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762383.363756:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762383.363765:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762383.363776:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048861952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762383.363787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762383.363790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762383.363792:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762383.363795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1832 00000100:00080000:3.0:1789762383.363798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762383.363802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762383.363806:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762383.363816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048862080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762383.363824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048861824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762383.363828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762383.363834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762383.363839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762383.363843:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1832 00000100:00080000:3.0:1789762383.363852:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762383.363856:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762383.363862:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762383.363884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048862208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762383.363893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762383.363897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762383.363900:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762383.363903:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1832 00000100:00080000:3.0:1789762383.363906:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762383.363910:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762383.363917:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789762383.363917:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762383.363933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048862336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762383.363989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762383/real 1789762383] req@ffff9811e1a22d80 x1876700048861824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762383.364017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048861824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762383.364063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762383.364069:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762383.364075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048861824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762383.364086:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048861952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762383.364134:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762383.364152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762383/real 1789762383] req@ffff9811e1a22680 x1876700048861952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762383.364170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048861952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762383.364181:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762383.364186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762383.364190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048861952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762383.364229:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762383.364272:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762383.364317:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762383/real 1789762383] req@ffff9811e1a21880 x1876700048862080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762383.364333:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048862080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762383.364344:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762383.364348:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762383.364353:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762383.364370:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762383.364396:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762383.364410:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762383/real 1789762383] req@ffff9811e1a21c00 x1876700048862208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762383.364426:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048862208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762383.364436:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762383.364440:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762383.364444:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762383.364451:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762383.364506:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762383.364520:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762383/real 1789762383] req@ffff9811e1a22a00 x1876700048862336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762383.364535:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048862336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762383.364545:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762383.364549:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762383.364553:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762383.619622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762384.259578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762384.643546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762385.283572:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762385.667536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762386.307580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762386.691602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762387.331578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762387.715553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762388.355606:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762388.483566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762388.483576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762388.483580:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762388.483591:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1838 00000100:00080000:2.0:1789762388.483604:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762388.483624:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762388.483676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048862464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762388.483709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762388.483713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762388.483715:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762388.483719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1838 00000100:00080000:2.0:1789762388.483723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762388.483727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762388.483743:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762388.483756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048862592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762388.483769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762388.483771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762388.483774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762388.483778:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1838 00000100:00080000:2.0:1789762388.483785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762388.483786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048862464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762388.483792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762388.483816:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762388.483828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048862720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762388.483836:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762388.483839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762388.483840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762388.483842:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1838 00000100:00080000:2.0:1789762388.483846:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762388.483849:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762388.483853:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762388.483859:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762388.483871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048862848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762388.483880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762388.483888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762388.483889:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762388.483893:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1838 00000100:00080000:2.0:1789762388.483896:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762388.483899:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762388.483903:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762388.483914:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762388/real 1789762388] req@ffff9811d863b800 x1876700048862464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762388.483920:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048862976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762388.483950:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048862464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762388.483959:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762388.483963:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762388.483966:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048862464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762388.483988:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048862592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762388.484015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762388.484030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762388/real 1789762388] req@ffff9811d863a300 x1876700048862592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762388.484051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048862592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762388.484058:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762388.484061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762388.484063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048862592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762388.484068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762388.484086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762388.484094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762388/real 1789762388] req@ffff9811d863b480 x1876700048862720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762388.484102:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048862720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762388.484120:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762388.484123:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762388.484126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762388.484130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762388.484146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762388.484152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762388/real 1789762388] req@ffff9811d863aa00 x1876700048862848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762388.484160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048862848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762388.484168:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762388.484172:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762388.484175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762388.484182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762388.484203:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762388.484214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762388/real 1789762388] req@ffff9811d8638000 x1876700048862976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762388.484225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048862976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762388.484230:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762388.484232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762388.484241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048862976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762388.739551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762389.379609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762389.763546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762390.403620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762390.787536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762391.427615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762391.811556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762392.451629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762392.835537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762393.475622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762393.603616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762393.603628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762393.603633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762393.603643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1843 00000100:00080000:1.0:1789762393.603654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762393.603664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762393.603703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048863104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762393.603740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762393.603744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762393.603746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762393.603751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1843 00000100:00080000:1.0:1789762393.603756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762393.603761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762393.603772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762393.603788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048863232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762393.603800:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048863104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762393.603807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762393.603816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762393.603818:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762393.603830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1843 00000100:00080000:1.0:1789762393.603835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762393.603875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762393.603883:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762393.603900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048863360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762393.603913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762393.603917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762393.603921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762393.603927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1843 00000100:00080000:1.0:1789762393.603936:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762393.603943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048863232:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762393.603945:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762393.603957:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762393.603981:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048863488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762393.603986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863360:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762393.604010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762393.604015:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789762393.604019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863488:192.168.203.113@tcp:8:kworker.0 00000100:00080000:1.0:1789762393.604021:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762393.604041:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1843 00000100:00080000:1.0:1789762393.604047:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762393.604056:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762393.604064:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762393.604085:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048863616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762393.604139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762393.859553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762394.499610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762394.883568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762395.523635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762395.907538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762396.547594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762396.675762:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762396.675814:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762396.675848:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762396.675885:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762396.675901:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762396.675920:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762393/real 1789762396] req@ffff9811c92c4380 x1876700048863232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789762396.675928:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762396.675942:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762396.675958:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048863232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762396.675974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762396.675980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762396.675985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048863232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762396.675995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762393/real 1789762396] req@ffff9811c92c4a80 x1876700048863360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762396.676009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048863360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762396.676019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762396.676023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762396.676027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762396.676042:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762393/real 1789762396] req@ffff9811c92c5f80 x1876700048863104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762396.676057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048863104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762396.676067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762396.676071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762396.676074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048863104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762396.676106:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762393/real 1789762396] req@ffff9811c92c6a00 x1876700048863616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762396.676121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048863616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762396.676131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762396.676135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762396.676138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762396.676146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762393/real 1789762396] req@ffff9811c92c6300 x1876700048863488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762396.676158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048863488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762396.676165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762396.676170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762396.676173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048863488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762396.931526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762397.571602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762397.955525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762398.595616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762398.723581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762398.723590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762398.723593:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762398.723602:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1848 00000100:00080000:3.0:1789762398.723611:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762398.723617:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762398.723647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048863744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762398.723670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762398.723672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762398.723674:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762398.723677:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1848 00000100:00080000:3.0:1789762398.723693:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762398.723700:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762398.723701:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048863744:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789762398.723708:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762398.723721:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048863872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762398.723731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762398.723734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762398.723735:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762398.723738:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1848 00000100:00080000:3.0:1789762398.723743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762398.723746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762398.723749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762398.723756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048864000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762398.723763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762398.723768:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762398.723768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762398.723774:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762398.723777:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1848 00000100:00080000:3.0:1789762398.723780:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762398.723783:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762398.723788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762398.723800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048864128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762398.723829:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762398.723834:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762398/real 1789762398] req@ffff9811e1a21180 x1876700048863744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762398.723835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762398.723840:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762398.723851:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1848 00000100:00100000:2.0:1789762398.723857:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048863744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762398.723857:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762398.723868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789762398.723871:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762398.723876:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:3.0:1789762398.723877:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762398.723880:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048863744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762398.723898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048864256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762398.723907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048863872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762398.723949:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762398.723965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762398/real 1789762398] req@ffff9811e1a22300 x1876700048863872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762398.723978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048863872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762398.723987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762398.723991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762398.724005:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048863872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762398.724011:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762398.724032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762398.724056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762398/real 1789762398] req@ffff9811e1a23100 x1876700048864000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762398.724069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048864000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762398.724094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762398.724097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762398.724100:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762398.724107:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762398.724137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762398.724150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762398/real 1789762398] req@ffff9811e1a23800 x1876700048864128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762398.724162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048864128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762398.724171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762398.724194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762398.724198:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762398.724215:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762398.724237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762398.724257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762398/real 1789762398] req@ffff9811e1a23480 x1876700048864256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762398.724269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048864256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762398.724286:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762398.724290:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762398.724293:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762398.979541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762399.619610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762400.003530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762400.643648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762401.027509:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762401.667595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762402.051522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762402.691655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762403.075520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762403.715668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762403.843544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762403.843555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762403.843559:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762403.843570:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1853 00000100:00080000:3.0:1789762403.843582:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762403.843591:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762403.843634:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048864384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762403.843663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762403.843666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762403.843679:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762403.843682:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1853 00000100:00080000:3.0:1789762403.843686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762403.843689:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762403.843700:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762403.843713:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048864512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762403.843723:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762403.843727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762403.843729:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762403.843733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1853 00000100:00080000:3.0:1789762403.843743:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762403.843746:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048864384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762403.843749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762403.843753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762403.843768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048864640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762403.843779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762403.843782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762403.843783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762403.843787:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1853 00000100:00080000:3.0:1789762403.843791:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762403.843794:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762403.843831:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762403.843876:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762403/real 1789762403] req@ffff9811e1a20000 x1876700048864384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762403.843878:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762403.843896:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048864384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762403.843902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048864768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762403.843915:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762403.843920:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762403.843926:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048864384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762403.843945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048864512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762403.843972:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762403.843988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762403/real 1789762403] req@ffff9811e1a20700 x1876700048864512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762403.844005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048864512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762403.844016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762403.844020:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762403.844025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048864512:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789762403.844032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762403.844036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762403.844038:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762403.844044:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1853 00000100:00080000:3.0:1789762403.844048:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762403.844051:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762403.844057:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762403.844062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762403.844071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048864896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762403.844092:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762403.844115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762403/real 1789762403] req@ffff9811e1a20380 x1876700048864640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762403.844167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048864640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762403.844178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762403.844189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762403.844193:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762403.844198:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762403.844217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762403.844226:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762403/real 1789762403] req@ffff9811e1a20e00 x1876700048864768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762403.844235:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048864768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762403.844241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762403.844243:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762403.844245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762403.844254:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762403.844265:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762403.844271:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762403/real 1789762403] req@ffff9811e1a21500 x1876700048864896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762403.844279:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048864896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762403.844294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762403.844296:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762403.844299:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048864896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762404.099507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762404.739630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762405.123503:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762405.763622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762406.147534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762406.787604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762407.171578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762407.811623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762408.195514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762408.835560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762408.963621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762408.963641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762408.963644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762408.963653:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1858 00000100:00080000:2.0:1789762408.963662:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762408.963668:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762408.963701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048865024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762408.963722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762408.963725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762408.963727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762408.963730:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1858 00000100:00080000:2.0:1789762408.963732:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762408.963734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762408.963742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762408.963756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048865152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762408.963768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762408.963771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762408.963790:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762408.963795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1858 00000100:00080000:2.0:1789762408.963799:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762408.963802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762408.963807:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762408.963819:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048865024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762408.963828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048865280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762408.963847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762408.963850:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762408.963852:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762408.963855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1858 00000100:00080000:2.0:1789762408.963858:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762408.963860:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762408.963864:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762408.963874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048865408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762408.963881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762408.963884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762408.963885:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762408.963890:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1858 00000100:00080000:2.0:1789762408.963894:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762408.963896:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762408.963898:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762408.963915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048865536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762408.963956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762408.964064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762408/real 1789762408] req@ffff9811d8638380 x1876700048865024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762408.964124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048865024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762408.964161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762408.964169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762408.964176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048865024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762408.964190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048865152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762408.964254:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762408.964275:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762408/real 1789762408] req@ffff9811d8638e00 x1876700048865152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762408.964291:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048865152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762408.964302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762408.964306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762408.964310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048865152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762408.964374:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762408.964431:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762408.964450:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762408/real 1789762408] req@ffff9811d8638700 x1876700048865280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762408.964463:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048865280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762408.964472:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762408.964476:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762408.964479:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762408.964486:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762408.964542:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762408.964562:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762408/real 1789762408] req@ffff9811d8638000 x1876700048865408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762408.964600:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048865408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762408.964616:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762408.964622:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762408.964629:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762408.964642:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762408.964677:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762408.964692:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762408/real 1789762408] req@ffff9811d863aa00 x1876700048865536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762408.964728:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048865536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762408.964743:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762408.964749:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762408.964756:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762409.219532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762409.859550:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762410.243560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762410.883579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762411.267536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762411.907532:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762412.291528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762412.931592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762413.315531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762413.955535:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762414.083567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762414.083579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762414.083583:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762414.083594:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1863 00000100:00080000:1.0:1789762414.083605:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762414.083612:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762414.083645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048865664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762414.083670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762414.083673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762414.083674:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762414.083677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1863 00000100:00080000:1.0:1789762414.083680:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762414.083682:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762414.083690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762414.083699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048865792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762414.083710:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048865664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762414.083728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762414.083731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762414.083733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762414.083737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1863 00000100:00080000:1.0:1789762414.083740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762414.083743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762414.083747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762414.083756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048865920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762414.083763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762414.083768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789762414.083769:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762414.083771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762414.083776:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1863 00000100:00080000:1.0:1789762414.083779:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762414.083782:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762414.083787:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762414.083796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048866048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762414.083803:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762414.083805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762414.083807:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762414.083810:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762414/real 1789762414] req@ffff9811c92c7100 x1876700048865664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762414.083813:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1863 00000100:00080000:1.0:1789762414.083823:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762414.083828:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762414.083833:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762414.083834:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048865664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762414.083844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048866176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762414.083845:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762414.083854:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762414.083861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048865664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762414.083883:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048865792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762414.083900:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762414.083908:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762414/real 1789762414] req@ffff9811c92c7800 x1876700048865792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762414.083917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048865792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762414.083922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762414.083925:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762414.083927:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048865792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762414.083938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762414.083952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762414.083959:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762414/real 1789762414] req@ffff9811c92c4700 x1876700048865920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762414.083967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048865920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762414.083972:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762414.083975:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762414.083977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048865920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762414.083980:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762414.083991:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762414.083996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762414/real 1789762414] req@ffff9811c92c6300 x1876700048866048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762414.084004:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048866048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762414.084018:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762414.084021:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762414.084023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762414.084026:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762414.084039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762414.084045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762414/real 1789762414] req@ffff9811c92c6a00 x1876700048866176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762414.084073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048866176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762414.084080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762414.084082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762414.084084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762414.339542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762414.979591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762415.363535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762416.003575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762416.387551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762417.027605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762417.411530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762418.051562:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762418.435560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762419.075665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762419.203577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762419.203602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762419.203606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762419.203614:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1868 00000100:00080000:1.0:1789762419.203623:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762419.203628:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762419.203661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048866304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762419.203686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762419.203690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762419.203691:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762419.203694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1868 00000100:00080000:1.0:1789762419.203696:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762419.203698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762419.203705:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762419.203713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048866432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762419.203722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762419.203740:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048866304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762419.203741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762419.203745:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762419.203752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1868 00000100:00080000:1.0:1789762419.203755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762419.203758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762419.203779:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762419.203792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048866560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762419.203807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762419.203812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762419.203815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762419.203820:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1868 00000800:00000100:2.0:1789762419.203841:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762419.203844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762419.203851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762419.203858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762419.203871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048866688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762419.203879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762419.203883:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762419/real 1789762419] req@ffff9811c92c5f80 x1876700048866304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762419.203884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762419.203887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762419.203894:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1868 00000100:00080000:1.0:1789762419.203900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762419.203904:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762419.203910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048866304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762419.203916:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762419.203926:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762419.203930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762419.203933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048866304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762419.203958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048866816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762419.203960:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048866432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762419.203987:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762419.203997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762419/real 1789762419] req@ffff9811c92c4a80 x1876700048866432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762419.204038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048866432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762419.204052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762419.204054:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762419.204057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048866432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762419.204078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762419.204097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762419.204109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762419/real 1789762419] req@ffff9811c92c4380 x1876700048866560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762419.204122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048866560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762419.204132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762419.204136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762419.204139:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762419.204146:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762419.204162:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762419.204170:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762419/real 1789762419] req@ffff9811c92c7480 x1876700048866688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762419.204179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048866688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762419.204184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762419.204186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762419.204194:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762419.204199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762419.204217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762419.204244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762419/real 1789762419] req@ffff9811c92c4e00 x1876700048866816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762419.204258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048866816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762419.204266:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762419.204269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762419.204271:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048866816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762419.459519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762420.099635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762420.483585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762421.123633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762421.507538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762422.147657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762422.531530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762423.171776:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762423.555570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762424.195633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762424.323650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762424.323668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762424.323675:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762424.323692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1873 00000100:00080000:1.0:1789762424.323740:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762424.323753:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762424.323802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048866944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762424.323852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762424.323857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762424.323860:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762424.323865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1873 00000100:00080000:1.0:1789762424.323870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762424.323875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762424.323898:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762424.323915:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048867072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762424.323961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762424.323965:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762424.323980:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762424.324003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048866944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762424.324021:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1873 00000100:00080000:1.0:1789762424.324050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762424.324058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762424.324066:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762424.324084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048867200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762424.324097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762424.324102:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762424.324104:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762424.324119:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1873 00000100:00080000:1.0:1789762424.324125:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762424.324128:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762424.324133:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762424.324146:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048867328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762424.324157:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762424.324170:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1789762424.324170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762424.324188:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762424.324196:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1873 00000100:00080000:1.0:1789762424.324201:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762424.324207:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762424.324216:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762424.324241:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048867456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762424.324300:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762424/real 1789762424] req@ffff9811c92c4000 x1876700048866944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762424.324329:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048866944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762424.324343:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762424.324348:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762424.324353:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048866944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762424.324421:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048867072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762424.324465:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762424.324487:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762424/real 1789762424] req@ffff9811c92c6680 x1876700048867072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762424.324603:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048867072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762424.324614:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762424.324618:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762424.324621:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048867072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762424.324629:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762424.324657:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762424.324670:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762424/real 1789762424] req@ffff9811c92c5c00 x1876700048867200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762424.324682:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048867200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762424.324690:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762424.324693:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762424.324696:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762424.324702:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762424.324730:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762424.324740:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762424/real 1789762424] req@ffff9811c92c7b80 x1876700048867328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762424.324762:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048867328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762424.324770:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762424.324773:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762424.324794:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762424.324800:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762424.324821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762424.324830:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762424/real 1789762424] req@ffff9811c92c5880 x1876700048867456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762424.324840:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048867456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762424.324847:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762424.324850:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762424.324853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762424.579647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762425.219684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762425.603586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762426.243584:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762426.627572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762427.267661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762427.651541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762428.291619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762428.675590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762429.315632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762429.443599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762429.443611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762429.443617:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762429.443628:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1879 00000100:00080000:3.0:1789762429.443640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762429.443649:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762429.443689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048867584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762429.443718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762429.443722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762429.443725:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762429.443734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1879 00000100:00080000:3.0:1789762429.443738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762429.443742:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762429.443752:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762429.443765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048867712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762429.443779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762429.443787:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048867584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762429.443788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762429.443811:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762429.443821:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1879 00000100:00080000:3.0:1789762429.443826:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762429.443830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762429.443836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762429.443852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048867840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762429.443862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762429.443866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762429.443868:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762429.443871:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1879 00000100:00080000:3.0:1789762429.443889:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762429.443893:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762429.443898:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762429.443914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048867968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762429.443916:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762429.443930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762429.443934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762429.443936:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762429.443941:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1879 00000100:00080000:3.0:1789762429.443970:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762429.443976:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762429.443982:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762429.444006:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048868096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762429.444015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762429/real 1789762429] req@ffff9811e1a21f80 x1876700048867584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762429.444044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048867584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762429.444056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762429.444061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762429.444066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048867584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762429.444105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048867712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762429.444133:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762429.444148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762429/real 1789762429] req@ffff9811e1a20a80 x1876700048867712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762429.444161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048867712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762429.444170:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762429.444173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762429.444177:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048867712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762429.444182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762429.444198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762429.444207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762429/real 1789762429] req@ffff9811e1a21500 x1876700048867840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762429.444238:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048867840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762429.444247:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762429.444250:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762429.444253:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762429.444259:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762429.444280:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762429.444290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762429/real 1789762429] req@ffff9811e1a20e00 x1876700048867968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762429.444302:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048867968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762429.444310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762429.444312:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762429.444315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048867968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762429.444320:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762429.444335:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762429.444343:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762429/real 1789762429] req@ffff9811e1a20380 x1876700048868096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762429.444354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048868096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762429.444374:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762429.444378:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762429.444381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762429.699630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762430.339665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762430.723546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762431.363639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762431.747581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762432.387659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762432.771571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762433.411701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762433.795601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762434.435722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762434.563718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762434.563733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762434.563739:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762434.563752:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1884 00000100:00080000:1.0:1789762434.563768:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762434.563777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762434.563839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048868224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762434.563892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762434.563897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762434.563899:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762434.563931:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1884 00000100:00080000:1.0:1789762434.563937:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762434.563954:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762434.563966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762434.563986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048868352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762434.563986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048868224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762434.564013:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762434.564017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762434.564020:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762434.564043:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1884 00000100:00080000:1.0:1789762434.564048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762434.564054:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762434.564084:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762434.564102:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762434.564106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048868480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762434.564124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762434.564129:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762434.564143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762434.564149:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1884 00000100:00080000:1.0:1789762434.564157:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762434.564164:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762434/real 1789762434] req@ffff9811c92c5500 x1876700048868224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762434.564175:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762434.564202:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048868608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762434.564213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048868224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762434.564223:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762434.564237:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762434.564243:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762434.564253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762434.564254:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1884 00000100:00080000:1.0:1789762434.564264:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762434.564271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762434.564277:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564286:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048868224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762434.564287:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762434.564314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048868736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762434.564354:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048868352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762434.564399:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564416:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762434/real 1789762434] req@ffff9811c92c6d80 x1876700048868352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762434.564448:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048868352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762434.564469:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762434.564473:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762434.564539:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048868352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762434.564549:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762434.564582:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564597:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762434/real 1789762434] req@ffff9811c92c5180 x1876700048868480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762434.564613:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048868480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762434.564624:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762434.564628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762434.564632:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762434.564641:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762434.564675:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564696:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762434/real 1789762434] req@ffff9811c92c5880 x1876700048868608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762434.564722:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048868608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762434.564740:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762434.564746:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762434.564764:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762434.564774:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762434.564805:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762434.564821:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762434/real 1789762434] req@ffff9811c92c7b80 x1876700048868736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762434.564837:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048868736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762434.564865:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762434.564868:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762434.564872:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048868736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762434.819578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762435.459649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762435.843614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762436.483755:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762436.867630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762437.507704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762437.891609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762438.531660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762438.915559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762439.555589:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762439.683613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762439.683622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762439.683626:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762439.683637:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1889 00000100:00080000:3.0:1789762439.683647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762439.683654:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762439.683688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048868864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762439.683715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762439.683719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762439.683721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762439.683724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1889 00000100:00080000:3.0:1789762439.683727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762439.683731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762439.683740:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762439.683751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048868992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762439.683763:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048868864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762439.683764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762439.683773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762439.683783:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762439.683788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1889 00000100:00080000:3.0:1789762439.683792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762439.683797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762439.683802:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762439.683830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048869120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762439.683839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762439.683842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762439.683843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762439.683849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1889 00000800:00000100:1.0:1789762439.683850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762439.683864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762439.683884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762439.683890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762439.683901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048869248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762439.683925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762439.683928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762439.683930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762439.683933:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1889 00000100:00080000:3.0:1789762439.683938:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762439.683939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762439/real 1789762439] req@ffff9811e1a20700 x1876700048868864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762439.683949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762439.683958:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762439.683961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048868864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762439.683975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762439.683979:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762439.683982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048868864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762439.683989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048869376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762439.684004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048868992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762439.684031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762439.684041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762439/real 1789762439] req@ffff9811e1a20000 x1876700048868992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762439.684050:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048868992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762439.684056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762439.684058:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762439.684061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048868992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762439.684064:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762439.684076:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762439.684083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762439/real 1789762439] req@ffff9811e1a23480 x1876700048869120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762439.684091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048869120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762439.684106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762439.684108:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762439.684111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762439.684115:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762439.684130:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762439.684137:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762439/real 1789762439] req@ffff9811e1a23800 x1876700048869248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762439.684144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048869248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762439.684150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762439.684151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762439.684153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762439.684163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762439.684175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762439.684181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762439/real 1789762439] req@ffff9811e1a23100 x1876700048869376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762439.684191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048869376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762439.684196:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762439.684198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762439.684205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762439.939648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762440.579627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762440.963570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762441.603735:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762441.987614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762442.627608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762443.011546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762443.651621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762444.035560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762444.675594:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762444.803619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762444.803629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762444.803633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762444.803653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1894 00000100:00080000:1.0:1789762444.803664:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762444.803670:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762444.803741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048869504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762444.803785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762444.803790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762444.803792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762444.803797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1894 00000100:00080000:1.0:1789762444.803801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762444.803804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762444.803814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762444.803846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048869632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762444.803885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762444.803888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762444.803890:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762444.803894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1894 00000100:00080000:1.0:1789762444.803897:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762444.803915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048869504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762444.803921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762444.803927:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762444.803939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048869760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762444.803949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762444.803952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762444.803954:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762444.803957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1894 00000100:00080000:1.0:1789762444.803961:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762444.803963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762444.803967:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762444.803976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048869888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762444.803982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762444.803992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762444.803996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762444.803997:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762444.804001:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1894 00000100:00080000:1.0:1789762444.804004:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762444.804007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762444.804010:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762444.804031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048870016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762444.804039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762444/real 1789762444] req@ffff9811c92c5c00 x1876700048869504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762444.804095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048869504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762444.804107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762444.804112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762444.804117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048869504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762444.804125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048869632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762444.804153:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762444.804166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762444/real 1789762444] req@ffff9811c92c6680 x1876700048869632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762444.804186:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048869632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762444.804195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762444.804198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762444.804202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048869632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762444.804231:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762444.804272:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762444.804292:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762444/real 1789762444] req@ffff9811c92c4000 x1876700048869760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762444.804330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048869760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762444.804373:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762444.804377:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762444.804381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762444.804387:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762444.804413:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762444.804425:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762444/real 1789762444] req@ffff9811c92c4e00 x1876700048869888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762444.804437:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048869888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762444.804445:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762444.804449:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762444.804452:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048869888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762444.804457:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762444.804509:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762444.804544:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762444/real 1789762444] req@ffff9811c92c7480 x1876700048870016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762444.804552:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048870016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762444.804558:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762444.804560:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762444.804569:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762445.059582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762445.699572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762446.083604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762446.723611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762447.107613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762447.747710:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762448.131577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762448.771695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762449.155604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762449.795821:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762449.923693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762449.923706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762449.923711:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762449.923735:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1899 00000100:00080000:2.0:1789762449.923749:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762449.923757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762449.923827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048870144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762449.923864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762449.923869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762449.923871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762449.923875:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1899 00000100:00080000:2.0:1789762449.923879:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762449.923883:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762449.923894:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762449.923907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048870272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762449.923920:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762449.923923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762449.923948:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762449.923952:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1899 00000100:00080000:2.0:1789762449.923957:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762449.923961:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762449.923968:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762449.923980:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048870400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762449.923992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762449.923996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762449.923998:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762449.924002:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1899 00000100:00080000:2.0:1789762449.924006:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762449.924012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048870144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762449.924014:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762449.924020:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762449.924033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048870528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762449.924046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762449.924050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762449.924052:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762449.924056:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1899 00000100:00080000:2.0:1789762449.924060:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762449.924064:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762449.924068:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762449.924091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048870656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762449.924115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762449.924166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762449/real 1789762449] req@ffff9811d863b480 x1876700048870144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762449.924188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048870144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762449.924201:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762449.924205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762449.924210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048870144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762449.924218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048870272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762449.924240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762449.924252:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762449/real 1789762449] req@ffff9811d863a300 x1876700048870272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762449.924265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048870272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762449.924274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762449.924277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762449.924281:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048870272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762449.924287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762449.924303:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762449.924320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762449/real 1789762449] req@ffff9811d863b800 x1876700048870400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762449.924350:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048870400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762449.924359:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762449.924363:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762449.924366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762449.924400:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762449.924431:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762449.924451:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762449/real 1789762449] req@ffff9811d8638a80 x1876700048870528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762449.924509:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048870528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762449.924525:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762449.924531:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762449.924537:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762449.924547:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762449.924584:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762449.924605:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762449/real 1789762449] req@ffff9811d863a680 x1876700048870656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762449.924626:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048870656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762449.924656:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762449.924670:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762449.924691:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048870656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762450.179570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762450.819830:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762451.203594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762451.843744:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762452.227642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762452.867786:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762453.251530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762453.891619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762454.275637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762454.915751:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762455.043746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762455.043771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762455.043776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762455.043788:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1904 00000100:00080000:3.0:1789762455.043801:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762455.043809:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762455.043873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048870784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762455.043911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762455.043923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762455.043925:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762455.043931:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1904 00000100:00080000:3.0:1789762455.043936:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762455.043942:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762455.043954:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762455.043969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048870912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762455.043977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048870784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762455.043985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762455.043990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762455.043991:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762455.043996:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1904 00000100:00080000:3.0:1789762455.043998:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762455.044022:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762455.044028:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762455.044039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048871040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762455.044050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762455.044054:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762455.044056:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762455.044062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762455.044064:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1904 00000100:00080000:3.0:1789762455.044073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762455.044079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762455.044087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762455.044103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048871168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762455.044115:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762455/real 1789762455] req@ffff9811e1a22300 x1876700048870784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762455.044120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762455.044145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762455.044149:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762455.044153:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1904 00000100:00080000:3.0:1789762455.044157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762455.044160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048870784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762455.044164:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762455.044173:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762455.044178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762455.044183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762455.044186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048871296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762455.044192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048870784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762455.044249:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048870912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762455.044279:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762455.044294:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762455/real 1789762455] req@ffff9811e1a21180 x1876700048870912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762455.044305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048870912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762455.044312:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762455.044315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762455.044317:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048870912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762455.044331:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762455.044350:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762455.044358:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762455/real 1789762455] req@ffff9811e1a22a00 x1876700048871040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762455.044368:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048871040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762455.044374:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762455.044376:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762455.044379:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762455.044383:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762455.044397:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762455.044404:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762455/real 1789762455] req@ffff9811e1a21c00 x1876700048871168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762455.044412:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048871168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762455.044418:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762455.044420:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762455.044422:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762455.044426:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762455.044440:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762455.044461:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762455/real 1789762455] req@ffff9811e1a21880 x1876700048871296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762455.044473:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048871296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762455.044478:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762455.044480:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762455.044482:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762455.299677:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762455.939608:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762456.323550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762456.963655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762457.347584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762457.987698:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762458.371594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762459.011690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762459.395669:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762460.035674:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762460.163562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762460.163588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762460.163592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762460.163600:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1909 00000100:00080000:3.0:1789762460.163610:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762460.163616:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762460.163649:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048871424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762460.163690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762460.163693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762460.163695:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762460.163699:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1909 00000100:00080000:3.0:1789762460.163702:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762460.163705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762460.163725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762460.163739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048871552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762460.163761:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048871424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762460.163762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762460.163768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762460.163770:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762460.163776:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1909 00000100:00080000:3.0:1789762460.163778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762460.163781:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762460.163784:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762460.163794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048871680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762460.163807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762460.163810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762460.163812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762460.163815:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1909 00000100:00080000:3.0:1789762460.163819:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762460.163821:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762460.163825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762460.163833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048871808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762460.163840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762460.163842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762460.163844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762460.163846:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1909 00000100:00080000:3.0:1789762460.163849:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762460.163851:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762460.163856:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762460.163863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048871936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762460.163872:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048871552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762460.163902:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762460.163922:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762460.163938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762460.419588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762461.059717:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762461.443543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762462.083615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762462.467590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762463.107685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789762463.235993:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789762463.236065:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762463.236087:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762463.236136:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762463.236149:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762463.236169:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762463.236182:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789762463.236221:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762460/real 1789762463] req@ffff9811e1a22a00 x1876700048871936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762463.236262:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048871936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762463.236277:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762463.236283:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762463.236311:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762463.236321:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762460/real 1789762463] req@ffff9811e1a22d80 x1876700048871552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762463.236336:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048871552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762463.236345:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762463.236349:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762463.236353:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048871552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762463.236360:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762460/real 1789762463] req@ffff9811e1a21880 x1876700048871680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762463.236372:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048871680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762463.236380:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762463.236383:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762463.236387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762463.236394:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762460/real 1789762463] req@ffff9811e1a21c00 x1876700048871808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762463.236407:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048871808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762463.236416:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762463.236419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762463.236430:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048871808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762463.236440:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762460/real 1789762463] req@ffff9811e1a22680 x1876700048871424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762463.236456:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048871424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762463.236513:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762463.236519:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762463.236524:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048871424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762463.491628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762464.131668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762464.515650:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762465.155699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762465.283587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762465.283599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762465.283603:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762465.283625:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1914 00000100:00080000:3.0:1789762465.283662:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762465.283700:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762465.283763:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048872064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762465.283804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762465.283808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762465.283810:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762465.283815:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1914 00000100:00080000:3.0:1789762465.283820:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762465.283824:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762465.283835:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762465.283857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048872192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762465.283879:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048872064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762465.283879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762465.283893:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762465.283899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762465.283907:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1914 00000100:00080000:3.0:1789762465.283912:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762465.283919:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762465.283929:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762465.283951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048872320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762465.283968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762465.283980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762465.283988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762465.283992:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762465.284001:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1914 00000100:00080000:3.0:1789762465.284011:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762465.284019:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762465.284029:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762465.284051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048872448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762465.284060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762465/real 1789762465] req@ffff9811e1a21180 x1876700048872064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762465.284093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762465.284094:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048872064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762465.284117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762465.284125:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762465.284137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762465.284139:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1914 00000100:00080000:3.0:1789762465.284151:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762465.284153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789762465.284166:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762465.284168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048872064:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789762465.284179:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762465.284207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048872576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762465.284214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048872192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762465.284280:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762465.284305:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762465/real 1789762465] req@ffff9811e1a22300 x1876700048872192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762465.284386:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048872192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762465.284400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762465.284406:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762465.284412:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048872192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762465.284423:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762465.284460:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762465.284534:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762465/real 1789762465] req@ffff9811e1a23100 x1876700048872320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762465.284554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048872320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762465.284567:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762465.284573:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762465.284578:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762465.284588:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762465.284623:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762465.284643:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762465/real 1789762465] req@ffff9811e1a23800 x1876700048872448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762465.284663:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048872448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762465.284721:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762465.284728:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762465.284734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762465.284761:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762465.284798:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762465.284819:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762465/real 1789762465] req@ffff9811e1a23480 x1876700048872576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762465.284838:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048872576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762465.284854:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762465.284860:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762465.284865:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762465.539665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762466.179768:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762466.563703:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762467.203833:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762467.587534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762468.227570:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762468.611522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762469.251673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762469.635550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762470.275585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762470.403604:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762470.403613:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762470.403617:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762470.403626:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1919 00000100:00080000:1.0:1789762470.403657:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762470.403665:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762470.403698:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048872704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762470.403722:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762470.403744:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762470.403746:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762470.403750:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1919 00000100:00080000:1.0:1789762470.403754:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762470.403757:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762470.403765:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762470.403769:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048872704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762470.403778:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048872832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762470.403788:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762470.403791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762470.403792:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762470.403795:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1919 00000100:00080000:1.0:1789762470.403798:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762470.403800:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762470.403803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762470.403811:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048872960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762470.403818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762470.403820:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762470.403821:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762470.403824:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1919 00000800:00000100:2.0:1789762470.403828:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762470.403829:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762470.403834:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762470.403844:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762470.403853:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048873088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762470.403860:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762470.403862:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762470.403864:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762470.403868:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1919 00000100:00100000:2.0:1789762470.403869:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762470/real 1789762470] req@ffff9811c92c4380 x1876700048872704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762470.403871:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762470.403878:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762470.403885:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762470.403894:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048872704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762470.403898:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048873216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762470.403922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762470.403926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762470.403930:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048872704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762470.403950:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048872832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762470.403967:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762470.403975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762470/real 1789762470] req@ffff9811c92c4a80 x1876700048872832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762470.403984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048872832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762470.403996:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762470.404001:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762470.404004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048872832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762470.404008:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762470.404019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762470.404035:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762470/real 1789762470] req@ffff9811c92c5f80 x1876700048872960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762470.404044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048872960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762470.404050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762470.404052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762470.404054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048872960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762470.404058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762470.404069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762470.404075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762470/real 1789762470] req@ffff9811c92c6a00 x1876700048873088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762470.404082:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048873088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762470.404087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762470.404089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762470.404091:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762470.404094:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762470.404104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762470.404109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762470/real 1789762470] req@ffff9811c92c6300 x1876700048873216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762470.404115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048873216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762470.404125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762470.404127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762470.404130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762470.659525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762471.299600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762471.683529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762472.323583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762472.707568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762473.347626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762473.731571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762474.371566:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762474.755564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762475.395585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762475.523580:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762475.523593:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762475.523613:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762475.523625:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1925 00000100:00080000:1.0:1789762475.523638:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762475.523647:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762475.523687:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048873344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762475.523719:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762475.523724:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762475.523727:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762475.523732:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1925 00000100:00080000:1.0:1789762475.523754:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762475.523761:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762475.523772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762475.523777:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048873344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762475.523795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048873472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762475.523815:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762475.523821:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762475.523823:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762475.523830:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1925 00000100:00080000:1.0:1789762475.523836:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762475.523843:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762475.523845:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762475.523855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762475.523872:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048873600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762475.523882:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762475.523886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762475/real 1789762475] req@ffff9811c92c4700 x1876700048873344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762475.523887:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762475.523892:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762475.523902:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1925 00000100:00080000:1.0:1789762475.523908:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762475.523912:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762475.523917:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762475.523918:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048873344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762475.523933:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762475.523934:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048873728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762475.523941:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762475.523951:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048873344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762475.523968:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762475.523971:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762475.523973:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762475.523979:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1925 00000100:00100000:2.0:1789762475.523981:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048873472:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762475.523985:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762475.523991:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762475.523997:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762475.524010:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048873856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762475.524019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762475.524048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762475/real 1789762475] req@ffff9811c92c7800 x1876700048873472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762475.524090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048873472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762475.524103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762475.524107:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762475.524111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048873472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762475.524118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762475.524142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762475.524154:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762475/real 1789762475] req@ffff9811c92c7100 x1876700048873600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762475.524167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048873600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762475.524175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762475.524179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762475.524183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762475.524190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762475.524206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762475.524216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762475/real 1789762475] req@ffff9811c92c6300 x1876700048873728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762475.524226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048873728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762475.524244:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762475.524247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762475.524250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762475.524269:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762475.524285:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762475.524293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762475/real 1789762475] req@ffff9811c92c6a00 x1876700048873856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762475.524303:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048873856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762475.524309:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762475.524312:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762475.524315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048873856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762475.779580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762476.419565:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762476.803555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762477.443572:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762477.827533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762478.467620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762478.851593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762479.491643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762479.875615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762480.515559:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762480.643610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762480.643620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762480.643625:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762480.643635:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1930 00000100:00080000:0.0:1789762480.643646:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762480.643654:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762480.643697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048873984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762480.643742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762480.643745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762480.643746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762480.643749:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1930 00000100:00080000:0.0:1789762480.643752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762480.643755:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762480.643766:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762480.643781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048874112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762480.643794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762480.643800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762480.643800:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048873984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762480.643803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762480.643814:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1930 00000100:00080000:0.0:1789762480.643818:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762480.643822:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762480.643826:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762480.643837:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048874240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762480.643845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762480.643847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762480.643849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762480.643851:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1930 00000100:00080000:0.0:1789762480.643869:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762480.643873:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762480.643878:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762480.643888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048874368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762480.643893:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762480.643901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762480.643904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762480.643905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762480.643908:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1930 00000100:00080000:0.0:1789762480.643911:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762480.643913:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762480.643916:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762480.643922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048874496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762480.643951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762480/real 1789762480] req@ffff9811c2e2a300 x1876700048873984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762480.643984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048873984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762480.643995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762480.643999:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762480.644004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048873984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762480.644031:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048874112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762480.644055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762480.644067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762480/real 1789762480] req@ffff9811c2e28000 x1876700048874112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762480.644079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048874112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762480.644088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762480.644092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762480.644095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048874112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762480.644100:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762480.644118:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762480.644128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762480/real 1789762480] req@ffff9811c2e29500 x1876700048874240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762480.644142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048874240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762480.644162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762480.644166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762480.644170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762480.644176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762480.644198:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762480.644207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762480/real 1789762480] req@ffff9811c2e2b480 x1876700048874368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762480.644218:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048874368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762480.644227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762480.644230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762480.644234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762480.644239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762480.644256:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762480.644265:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762480/real 1789762480] req@ffff9811c2e2b800 x1876700048874496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762480.644277:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048874496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762480.644284:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762480.644287:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762480.644296:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762480.899625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762481.539644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762481.923593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762482.563587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762482.947536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762483.587619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762483.971530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762484.611592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762484.995569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762485.635609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762485.763545:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762485.763556:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762485.763560:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762485.763571:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1935 00000100:00080000:0.0:1789762485.763582:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762485.763588:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762485.763617:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048874624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762485.763638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762485.763650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762485.763653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762485.763660:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1935 00000100:00080000:0.0:1789762485.763666:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762485.763672:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762485.763681:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762485.763689:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048874624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762485.763703:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048874752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762485.763722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762485.763725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762485.763727:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762485.763733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1935 00000100:00080000:0.0:1789762485.763736:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762485.763740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762485.763746:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762485.763756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048874880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762485.763775:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762485.763778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762485.763782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762485.763783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762485.763788:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1935 00000100:00080000:0.0:1789762485.763792:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762485.763795:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762485.763805:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762485.763818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048875008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762485.763826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762485.763829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762485.763830:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762485.763836:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762485/real 1789762485] req@ffff9811c2e29f80 x1876700048874624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762485.763839:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1935 00000100:00080000:0.0:1789762485.763848:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762485.763858:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762485.763869:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762485.763886:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048874624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762485.763897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048875136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762485.763903:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762485.763912:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762485.763922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048874624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762485.763956:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048874752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762485.763986:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762485.763996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762485/real 1789762485] req@ffff9811c2e28a80 x1876700048874752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762485.764005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048874752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762485.764011:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762485.764025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762485.764028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048874752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762485.764033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762485.764049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762485.764056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762485/real 1789762485] req@ffff9811c2e28380 x1876700048874880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762485.764063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048874880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762485.764068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762485.764070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762485.764080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048874880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762485.764086:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762485.764112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762485.764125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762485/real 1789762485] req@ffff9811c2e2aa00 x1876700048875008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762485.764137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048875008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762485.764146:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762485.764149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762485.764153:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762485.764159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762485.764201:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762485.764210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762485/real 1789762485] req@ffff9811c2e2b100 x1876700048875136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762485.764219:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048875136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762485.764224:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762485.764226:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762485.764228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762486.019565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762486.659661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762487.043630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762487.683650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762488.067562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762488.707636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762489.091578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762489.731611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762490.115541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762490.755610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762490.883593:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762490.883605:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762490.883610:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762490.883621:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1940 00000100:00080000:1.0:1789762490.883631:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762490.883639:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762490.883675:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048875264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762490.883703:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762490.883708:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762490.883710:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762490.883714:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1940 00000100:00080000:1.0:1789762490.883717:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762490.883721:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762490.883743:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762490.883757:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048875392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762490.883761:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048875264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762490.883778:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762490.883781:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762490.883784:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762490.883789:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1940 00000100:00080000:1.0:1789762490.883793:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762490.883798:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762490.883804:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762490.883815:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048875520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762490.883824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762490.883829:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762490.883834:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762490.883836:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762490.883842:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1940 00000100:00080000:1.0:1789762490.883845:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762490.883849:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762490.883855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762490.883863:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762490/real 1789762490] req@ffff9811c92c5f80 x1876700048875264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762490.883869:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048875648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762490.883885:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762490.883889:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048875264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762490.883891:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762490.883901:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762490.883902:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762490.883909:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762490.883913:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1940 00000100:00080000:1.0:1789762490.883918:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762490.883923:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048875264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762490.883927:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762490.883935:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762490.883947:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048875776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762490.883956:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048875392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762490.883982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762490.883991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762490/real 1789762490] req@ffff9811c92c4a80 x1876700048875392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762490.884001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048875392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762490.884006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762490.884009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762490.884011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048875392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762490.884016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762490.884026:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762490.884032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762490/real 1789762490] req@ffff9811c92c4380 x1876700048875520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762490.884040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048875520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762490.884045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762490.884046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762490.884048:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762490.884051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762490.884061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762490.884074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762490/real 1789762490] req@ffff9811c92c7480 x1876700048875648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762490.884082:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048875648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762490.884087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762490.884089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762490.884091:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762490.884102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762490.884113:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762490.884119:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762490/real 1789762490] req@ffff9811c92c4e00 x1876700048875776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762490.884126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048875776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762490.884132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762490.884134:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762490.884135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048875776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762491.139575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762491.779608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762492.163542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762492.803612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762493.187559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762493.827566:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762494.211572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762494.851576:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762495.235543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762495.875597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762496.003583:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762496.003600:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762496.003604:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762496.003614:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1945 00000100:00080000:0.0:1789762496.003625:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762496.003631:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762496.003662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048875904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762496.003684:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762496.003687:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762496.003689:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762496.003707:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1945 00000100:00080000:0.0:1789762496.003711:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762496.003715:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762496.003723:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762496.003737:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048876032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762496.003737:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048875904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762496.003751:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762496.003754:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762496.003758:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762496.003765:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1945 00000100:00080000:0.0:1789762496.003767:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762496.003769:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762496.003772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762496.003781:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048876160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762496.003787:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762496.003790:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762496.003791:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762496.003794:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1945 00000100:00080000:0.0:1789762496.003797:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762496.003800:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762496.003803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762496.003810:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048876288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762496.003824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762496.003825:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762496.003830:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762496.003832:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762496.003836:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1945 00000100:00080000:0.0:1789762496.003839:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762496.003841:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762496.003845:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762496.003854:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048876416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762496.003886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762496/real 1789762496] req@ffff9811c2e2ad80 x1876700048875904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762496.003915:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048875904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762496.003927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762496.003931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762496.003936:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048875904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762496.003958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048876032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762496.003982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762496.003995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762496/real 1789762496] req@ffff9811c2e29880 x1876700048876032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762496.004010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048876032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762496.004019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762496.004023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762496.004027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048876032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762496.004033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762496.004053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762496.004064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762496/real 1789762496] req@ffff9811c2e2bb80 x1876700048876160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762496.004089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048876160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762496.004098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762496.004102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762496.004106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762496.004112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762496.004132:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762496.004143:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762496/real 1789762496] req@ffff9811c2e29180 x1876700048876288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762496.004156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048876288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762496.004165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762496.004168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762496.004172:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762496.004178:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762496.004197:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762496.004208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762496/real 1789762496] req@ffff9811c2e28e00 x1876700048876416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762496.004221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048876416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762496.004242:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762496.004246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762496.004250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762496.259550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762496.899585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762497.283526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762497.923604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762498.307577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762498.947602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762499.331551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762499.971623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762500.355558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762500.995625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762501.123620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762501.123637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762501.123644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762501.123660:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1950 00000100:00080000:0.0:1789762501.123677:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762501.123690:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762501.123745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048876544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762501.123814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762501.123822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762501.123825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762501.123834:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1950 00000100:00080000:0.0:1789762501.123843:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762501.123851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762501.123905:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762501.123907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048876544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762501.123941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048876672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762501.123970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762501.123977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762501.123982:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762501.123990:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1950 00000800:00000100:2.0:1789762501.123998:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762501.124002:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762501.124016:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762501.124027:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762501.124054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048876800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762501.124060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762501/real 1789762501] req@ffff9811c2e29c00 x1876700048876544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762501.124112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762501.124120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048876544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762501.124124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762501.124132:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762501.124147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762501.124150:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1950 00000100:00080000:2.0:1789762501.124157:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762501.124165:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762501.124166:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048876544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762501.124185:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762501.124198:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762501.124210:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048876672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762501.124229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048876928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762501.124256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762501.124266:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762501.124269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762501.124287:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762501.124298:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1950 00000100:00080000:0.0:1789762501.124309:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762501.124321:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762501/real 1789762501] req@ffff9811c2e2a680 x1876700048876672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762501.124323:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762501.124345:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762501.124361:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048876672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762501.124381:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048877056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762501.124399:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762501.124432:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762501.124441:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048876672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762501.124456:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762501.124553:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762501.124574:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762501/real 1789762501] req@ffff9811c2e28700 x1876700048876800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762501.124590:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048876800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762501.124600:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762501.124604:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762501.124608:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762501.124627:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762501.124652:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762501.124664:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762501/real 1789762501] req@ffff9811c2e29c00 x1876700048876928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762501.124677:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048876928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762501.124686:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762501.124689:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762501.124703:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048876928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762501.124711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762501.124734:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762501.124745:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762501/real 1789762501] req@ffff9811c2e28e00 x1876700048877056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762501.124759:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048877056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762501.124768:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762501.124771:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762501.124775:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762501.379585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762502.019611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762502.403550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762503.043719:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762503.427571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762504.067622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762504.451562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762505.091631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762505.475558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762506.115657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762506.243544:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762506.243555:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762506.243559:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762506.243570:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1955 00000100:00080000:1.0:1789762506.243581:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762506.243588:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762506.243632:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048877184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762506.243657:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762506.243661:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762506.243662:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762506.243688:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1955 00000100:00080000:1.0:1789762506.243691:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762506.243694:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762506.243703:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762506.243713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048877312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762506.243723:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762506.243725:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762506.243727:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762506.243729:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1955 00000100:00080000:1.0:1789762506.243734:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762506.243737:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762506.243740:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762506.243747:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048877440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762506.243753:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762506.243755:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762506.243757:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762506.243760:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1955 00000100:00080000:1.0:1789762506.243762:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762506.243765:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762506.243769:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762506.243777:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048877184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762506.243781:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048877568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762506.243796:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762506.243800:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762506.243801:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762506.243806:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1955 00000100:00080000:1.0:1789762506.243809:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762506.243813:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762506.243817:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762506.243829:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048877696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762506.243847:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762506.243888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762506/real 1789762506] req@ffff9811c92c4000 x1876700048877184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762506.243904:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048877184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762506.243913:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762506.243917:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762506.243929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048877184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762506.243935:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048877312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762506.243952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762506.243975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762506/real 1789762506] req@ffff9811c92c6680 x1876700048877312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762506.243983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048877312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762506.243989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762506.243992:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762506.243994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048877312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762506.244004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762506.244017:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762506.244023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762506/real 1789762506] req@ffff9811c92c5c00 x1876700048877440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762506.244031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048877440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762506.244036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762506.244039:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762506.244041:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762506.244067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762506.244088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762506.244100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762506/real 1789762506] req@ffff9811c92c7b80 x1876700048877568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762506.244113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048877568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762506.244132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762506.244135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762506.244137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762506.244142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762506.244159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762506.244166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762506/real 1789762506] req@ffff9811c92c5880 x1876700048877696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762506.244174:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048877696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762506.244179:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762506.244181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762506.244183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048877696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762506.499558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762507.139590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762507.523521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762508.163684:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762508.547528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762509.187632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762509.571558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762510.211595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762510.595537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762511.235639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762511.363589:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762511.363598:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762511.363603:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762511.363615:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1960 00000100:00080000:1.0:1789762511.363629:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762511.363637:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762511.363679:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048877824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762511.363705:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762511.363708:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762511.363725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762511.363729:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1960 00000100:00080000:1.0:1789762511.363732:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762511.363737:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762511.363749:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762511.363763:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048877952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762511.363775:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762511.363775:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048877824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762511.363786:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762511.363792:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762511.363803:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1960 00000100:00080000:1.0:1789762511.363806:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762511.363808:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762511.363812:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762511.363820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048878080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762511.363827:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762511.363829:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762511.363831:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762511.363835:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1960 00000100:00080000:1.0:1789762511.363839:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762511.363842:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762511.363845:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762511.363865:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048878208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762511.363879:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762511.363882:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762511.363886:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762511.363888:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762511.363894:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1960 00000100:00080000:1.0:1789762511.363901:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762511.363904:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762511.363909:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762511.363921:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048878336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762511.363946:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762511/real 1789762511] req@ffff9811c92c5180 x1876700048877824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762511.363973:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048877824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762511.363985:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762511.363990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762511.363994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048877824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762511.364037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048877952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762511.364068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762511.364087:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762511/real 1789762511] req@ffff9811c92c6d80 x1876700048877952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762511.364109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048877952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762511.364120:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762511.364125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762511.364130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048877952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762511.364145:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762511.364176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762511.364190:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762511/real 1789762511] req@ffff9811c92c5500 x1876700048878080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762511.364202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048878080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762511.364211:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762511.364215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762511.364218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762511.364226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762511.364244:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762511.364254:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762511/real 1789762511] req@ffff9811c92c5880 x1876700048878208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762511.364275:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048878208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762511.364284:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762511.364288:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762511.364291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762511.364297:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762511.364317:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762511.364326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762511/real 1789762511] req@ffff9811c92c7b80 x1876700048878336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762511.364355:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048878336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762511.364363:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762511.364366:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762511.364370:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762511.619553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762512.259631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762512.643563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762513.283741:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762513.667598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762514.307642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762514.691560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762515.331626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762515.715551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762516.355679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762516.483604:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762516.483613:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762516.483617:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762516.483627:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1966 00000100:00080000:2.0:1789762516.483637:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762516.483644:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762516.483679:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048878464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762516.483715:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762516.483718:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762516.483720:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762516.483723:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1966 00000100:00080000:2.0:1789762516.483727:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762516.483731:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762516.483740:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762516.483751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048878464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762516.483753:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048878592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762516.483768:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762516.483771:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762516.483773:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762516.483777:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1966 00000100:00080000:2.0:1789762516.483781:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762516.483784:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762516.483787:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762516.483795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048878720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762516.483802:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762516.483804:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762516.483806:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762516.483809:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1966 00000100:00080000:2.0:1789762516.483817:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762516.483820:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762516.483826:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762516.483827:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762516.483839:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048878848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762516.483847:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762516.483850:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762516.483852:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762516.483854:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1966 00000100:00080000:2.0:1789762516.483857:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762516.483859:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762516.483863:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762516.483869:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762516/real 1789762516] req@ffff9811d8639f80 x1876700048878464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762516.483873:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048878976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762516.483910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048878464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762516.483920:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762516.483924:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762516.483945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048878464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762516.483977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048878592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762516.484003:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762516.484015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762516/real 1789762516] req@ffff9811d8639500 x1876700048878592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762516.484024:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048878592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762516.484031:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762516.484033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762516.484036:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048878592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762516.484040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762516.484054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762516.484061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762516/real 1789762516] req@ffff9811d8639880 x1876700048878720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762516.484069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048878720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762516.484074:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762516.484076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762516.484078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762516.484082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762516.484093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762516.484108:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762516/real 1789762516] req@ffff9811d863ad80 x1876700048878848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762516.484116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048878848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762516.484122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762516.484124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762516.484127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762516.484130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762516.484142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762516.484148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762516/real 1789762516] req@ffff9811d8639c00 x1876700048878976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762516.484156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048878976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762516.484161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762516.484163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762516.484165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048878976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762516.739539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762517.379600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762517.763568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762518.403643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762518.787539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762519.427603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762519.811568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762520.451713:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762520.835555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762521.475558:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762521.603553:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762521.603561:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762521.603565:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762521.603574:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1971 00000100:00080000:2.0:1789762521.603583:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762521.603589:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762521.603622:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048879104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762521.603643:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762521.603659:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762521.603661:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762521.603663:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1971 00000100:00080000:2.0:1789762521.603666:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762521.603669:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762521.603676:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762521.603685:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048879232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762521.603692:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762521.603694:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762521.603696:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762521.603698:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1971 00000100:00080000:2.0:1789762521.603701:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762521.603704:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762521.603707:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762521.603712:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048879360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762521.603719:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762521.603723:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789762521.603725:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048879104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762521.603725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762521.603733:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1971 00000100:00080000:2.0:1789762521.603736:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762521.603738:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762521.603746:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762521.603758:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048879488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762521.603769:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762521.603772:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762521.603774:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762521.603778:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1971 00000100:00080000:2.0:1789762521.603783:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762521.603785:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762521.603791:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762521.603798:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762521.603812:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048879616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762521.603825:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762521/real 1789762521] req@ffff9811d863b100 x1876700048879104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762521.603840:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048879104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762521.603855:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762521.603858:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762521.603862:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048879104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762521.603867:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048879232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762521.603884:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762521.603892:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762521/real 1789762521] req@ffff9811d863bb80 x1876700048879232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762521.603900:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048879232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762521.603906:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762521.603909:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762521.603911:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048879232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762521.603928:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762521.603939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762521.603944:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762521/real 1789762521] req@ffff9811d8639180 x1876700048879360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762521.603952:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048879360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762521.603957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762521.603959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762521.603961:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762521.603970:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762521.603993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762521.604001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762521/real 1789762521] req@ffff9811d8639c00 x1876700048879488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762521.604009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048879488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762521.604015:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762521.604017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762521.604019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762521.604022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762521.604032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762521.604037:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762521/real 1789762521] req@ffff9811d863ad80 x1876700048879616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762521.604044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048879616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762521.604049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762521.604051:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762521.604053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048879616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762521.859533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762522.499605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762522.883536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762523.523534:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762523.907600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762524.547648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762524.931575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762525.571618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762525.955546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762526.595621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762526.723597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762526.723607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762526.723611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762526.723622:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1976 00000100:00080000:3.0:1789762526.723634:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762526.723642:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762526.723686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048879744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762526.723715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762526.723718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762526.723719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762526.723727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1976 00000100:00080000:3.0:1789762526.723730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762526.723732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762526.723742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762526.723754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048879872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762526.723764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048879744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762526.723790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762526.723793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762526.723795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762526.723799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1976 00000100:00080000:3.0:1789762526.723802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762526.723805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762526.723809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762526.723818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048880000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762526.723825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762526.723828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762526.723829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762526.723831:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1976 00000100:00080000:3.0:1789762526.723833:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762526.723836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762526.723840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762526.723847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048880128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762526.723852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762526.723855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762526.723857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762526.723860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1976 00000100:00080000:3.0:1789762526.723871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762526.723875:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762526.723885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762526.723887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048879872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762526.723899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048880256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762526.723938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762526.723973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762526.723994:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762526.979657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762527.619628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762528.003538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762528.643606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762529.027554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762529.667621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789762529.795707:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789762529.795754:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762529.795771:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762529.795808:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762529.795818:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762529.795829:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762529.795838:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789762529.795858:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762526/real 1789762529] req@ffff9811e1a21500 x1876700048880256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762529.795887:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048880256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762529.795900:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762529.795905:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762529.795910:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762529.795919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762526/real 1789762529] req@ffff9811e1a20700 x1876700048879872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762529.795933:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048879872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762529.795942:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762529.795946:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762529.795950:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048879872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762529.795957:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762526/real 1789762529] req@ffff9811e1a20380 x1876700048880000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762529.795970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048880000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762529.795995:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762529.796000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762529.796004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762529.796012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762526/real 1789762529] req@ffff9811e1a20e00 x1876700048880128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762529.796026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048880128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762529.796034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762529.796038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762529.796042:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762529.796049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762526/real 1789762529] req@ffff9811e1a20000 x1876700048879744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762529.796063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048879744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762529.796072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762529.796076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762529.796080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048879744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762530.051527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762530.691631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762531.075533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762531.715768:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762531.843561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762531.843572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762531.843577:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762531.843587:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1981 00000100:00080000:1.0:1789762531.843617:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762531.843625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762531.843665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048880384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762531.843697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762531.843701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762531.843703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762531.843707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1981 00000100:00080000:1.0:1789762531.843711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762531.843716:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762531.843728:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762531.843743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048880512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762531.843754:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048880384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762531.843761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762531.843767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762531.843769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762531.843783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1981 00000100:00080000:1.0:1789762531.843788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762531.843794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762531.843800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762531.843818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048880640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762531.843829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762531.843840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789762531.843843:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762531.843846:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762531.843856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1981 00000100:00080000:1.0:1789762531.843866:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762531.843873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762531.843883:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762531.843897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762531/real 1789762531] req@ffff9811c92c5c00 x1876700048880384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762531.843905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048880768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762531.843920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762531.843932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762531.843934:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762531.843939:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1981 00000100:00080000:1.0:1789762531.843943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762531.843953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762531.843967:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762531.843994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048880896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762531.843995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048880384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762531.844037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762531.844045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762531.844051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048880384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762531.844087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048880512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762531.844117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762531.844132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762531/real 1789762531] req@ffff9811c92c6680 x1876700048880512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762531.844147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048880512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762531.844156:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762531.844159:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762531.844163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048880512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762531.844170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762531.844210:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762531.844222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762531/real 1789762531] req@ffff9811c92c4000 x1876700048880640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762531.844234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048880640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762531.844253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762531.844257:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762531.844260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762531.844266:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762531.844286:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762531.844298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762531/real 1789762531] req@ffff9811c92c4e00 x1876700048880768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762531.844309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048880768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762531.844317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762531.844320:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762531.844323:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762531.844328:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762531.844347:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762531.844359:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762531/real 1789762531] req@ffff9811c92c7480 x1876700048880896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762531.844372:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048880896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762531.844380:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762531.844384:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762531.844394:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048880896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762532.099516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762532.739648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762533.123531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762533.763577:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762534.147511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762534.787639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762535.171515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762535.811579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762536.195522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762536.835591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762536.963557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762536.963567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762536.963571:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762536.963581:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1986 00000100:00080000:3.0:1789762536.963618:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762536.963632:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762536.963672:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048881024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762536.963698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762536.963701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762536.963702:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762536.963705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1986 00000100:00080000:3.0:1789762536.963708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762536.963710:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762536.963719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762536.963728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048881152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762536.963737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762536.963741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762536.963742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762536.963744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1986 00000100:00080000:3.0:1789762536.963746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762536.963750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762536.963753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762536.963763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048881280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762536.963771:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048881024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762536.963773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762536.963788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762536.963790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762536.963795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1986 00000100:00080000:3.0:1789762536.963797:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762536.963800:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762536.963806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762536.963823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048881408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762536.963831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762536.963834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762536.963836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762536.963839:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1986 00000100:00080000:3.0:1789762536.963844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762536.963847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762536.963850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762536.963861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048881536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762536.963863:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762536.963933:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762536/real 1789762536] req@ffff9811e1a20a80 x1876700048881024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762536.963955:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048881024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762536.963967:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762536.963972:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762536.963976:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048881024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762536.963985:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048881152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762536.964010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762536.964023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762536/real 1789762536] req@ffff9811e1a21f80 x1876700048881152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762536.964037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048881152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762536.964045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762536.964049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762536.964053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048881152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762536.964078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762536.964098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762536.964126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762536/real 1789762536] req@ffff9811e1a20000 x1876700048881280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762536.964157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048881280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762536.964167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762536.964171:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762536.964175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762536.964182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762536.964204:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762536.964216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762536/real 1789762536] req@ffff9811e1a20e00 x1876700048881408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762536.964230:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048881408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762536.964238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762536.964241:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762536.964245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762536.964251:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762536.964272:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762536.964283:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762536/real 1789762536] req@ffff9811e1a20380 x1876700048881536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762536.964295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048881536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762536.964311:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762536.964315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762536.964319:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762537.219519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762537.859659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762538.243502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762538.883608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762539.267555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762539.907647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762540.291583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762540.931658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762541.315601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762541.955635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762542.083590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762542.083602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762542.083607:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762542.083618:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1991 00000100:00080000:1.0:1789762542.083629:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762542.083654:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762542.083687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048881664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762542.083717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762542.083722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762542.083724:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762542.083728:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1991 00000100:00080000:1.0:1789762542.083733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762542.083737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762542.083746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762542.083759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048881792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762542.083773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762542.083777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762542.083779:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762542.083783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1991 00000100:00080000:1.0:1789762542.083787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762542.083792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762542.083800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762542.083811:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048881664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762542.083817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048881920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762542.083833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762542.083837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762542.083845:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762542.083850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1991 00000100:00080000:1.0:1789762542.083856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762542.083861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762542.083867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762542.083882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048882048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762542.083894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762542.083898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762542.083900:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762542.083904:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1991 00000100:00080000:1.0:1789762542.083909:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762542.083918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789762542.083921:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762542.083927:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762542.083944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048882176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762542.083986:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762542/real 1789762542] req@ffff9811c92c4380 x1876700048881664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762542.084010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048881664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762542.084021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762542.084027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762542.084031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048881664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762542.084041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048881792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762542.084069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762542.084082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762542/real 1789762542] req@ffff9811c92c4a80 x1876700048881792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762542.084095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048881792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762542.084121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762542.084125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762542.084128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048881792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762542.084173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762542.084191:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762542.084200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762542/real 1789762542] req@ffff9811c92c5f80 x1876700048881920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762542.084208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048881920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762542.084214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762542.084216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762542.084219:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048881920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762542.084223:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762542.084235:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762542.084241:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762542/real 1789762542] req@ffff9811c92c6a00 x1876700048882048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762542.084248:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048882048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762542.084253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762542.084255:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762542.084257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762542.084260:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762542.084270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762542.084275:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762542/real 1789762542] req@ffff9811c92c6300 x1876700048882176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762542.084289:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048882176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762542.084294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762542.084296:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762542.084298:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762542.339529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762542.979635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762543.363568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762544.003557:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762544.387554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762545.027580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762545.411616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762546.051585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762546.435543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762547.075599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762547.203581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762547.203595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762547.203600:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762547.203613:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 1996 00000100:00080000:1.0:1789762547.203627:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762547.203637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762547.203705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048882304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762547.203740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762547.203744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762547.203746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762547.203750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 1996 00000100:00080000:1.0:1789762547.203754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762547.203759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762547.203770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762547.203784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048882432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762547.203788:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048882304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762547.203805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762547.203809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762547.203811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762547.203816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1996 00000100:00080000:1.0:1789762547.203819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762547.203824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762547.203829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762547.203840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048882560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762547.203850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762547.203853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762547.203855:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762547.203866:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1996 00000100:00080000:1.0:1789762547.203871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762547.203875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762547.203879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762547.203887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762547.203902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048882688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762547.203913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762547.203916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762547.203918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762547.203922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 1996 00000100:00100000:2.0:1789762547.203925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762547/real 1789762547] req@ffff9811c92c7100 x1876700048882304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762547.203928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762547.203938:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762547.203945:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762547.203950:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048882304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762547.203958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048882816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762547.203961:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762547.203968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762547.203974:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048882304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762547.204019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048882432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762547.204039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762547.204049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762547/real 1789762547] req@ffff9811c92c7800 x1876700048882432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762547.204059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048882432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762547.204093:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762547.204096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762547.204099:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048882432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762547.204106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762547.204140:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762547.204149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762547/real 1789762547] req@ffff9811c92c4700 x1876700048882560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762547.204163:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048882560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762547.204173:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762547.204176:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762547.204180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762547.204187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762547.204214:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762547.204224:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762547/real 1789762547] req@ffff9811c92c6300 x1876700048882688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762547.204234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048882688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762547.204240:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762547.204242:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762547.204250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762547.204256:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762547.204277:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762547.204288:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762547/real 1789762547] req@ffff9811c92c6a00 x1876700048882816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762547.204297:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048882816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762547.204305:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762547.204309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762547.204311:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048882816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762547.459563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762548.099591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762548.483581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762549.123636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762549.507566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762550.147619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762550.531510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762551.171632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762551.555524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762552.195586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762552.323589:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762552.323598:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762552.323603:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762552.323613:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2001 00000100:00080000:1.0:1789762552.323622:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762552.323629:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762552.323661:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048882944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762552.323691:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762552.323694:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762552.323695:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762552.323714:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2001 00000100:00080000:1.0:1789762552.323717:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762552.323720:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762552.323729:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762552.323739:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048883072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762552.323750:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762552.323750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048882944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762552.323759:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762552.323761:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762552.323765:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2001 00000100:00080000:1.0:1789762552.323767:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762552.323769:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762552.323772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762552.323780:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048883200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762552.323802:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762552.323804:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762552.323806:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762552.323809:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2001 00000100:00080000:1.0:1789762552.323817:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762552.323820:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762552.323825:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762552.323835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048883328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762552.323839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762552.323864:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762552.323868:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762552.323870:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762552.323874:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2001 00000100:00080000:1.0:1789762552.323877:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762552.323881:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762552.323885:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762552.323896:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048883456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762552.323905:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762552/real 1789762552] req@ffff9811c92c5f80 x1876700048882944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762552.323944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048882944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762552.323953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762552.323956:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762552.323959:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048882944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762552.323995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048883072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762552.324015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762552.324037:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762552/real 1789762552] req@ffff9811c92c4a80 x1876700048883072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762552.324048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048883072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762552.324055:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762552.324058:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762552.324060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048883072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762552.324065:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762552.324080:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762552.324088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762552/real 1789762552] req@ffff9811c92c4380 x1876700048883200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762552.324102:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048883200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762552.324110:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762552.324115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762552.324119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762552.324125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762552.324146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762552.324155:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762552/real 1789762552] req@ffff9811c92c7480 x1876700048883328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762552.324172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048883328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762552.324178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762552.324180:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762552.324182:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762552.324187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762552.324201:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762552.324208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762552/real 1789762552] req@ffff9811c92c4e00 x1876700048883456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762552.324216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048883456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762552.324221:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762552.324222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762552.324224:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762552.579631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762553.219622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762553.603583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762554.243560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762554.627547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762555.267563:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762555.651560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762556.291648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762556.675591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762557.315612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762557.443596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762557.443606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762557.443611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762557.443621:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2007 00000100:00080000:3.0:1789762557.443633:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762557.443641:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762557.443683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048883584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762557.443709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762557.443712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762557.443731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762557.443736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2007 00000100:00080000:3.0:1789762557.443740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762557.443744:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762557.443754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762557.443766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048883712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762557.443773:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048883584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762557.443789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762557.443793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762557.443795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762557.443800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2007 00000100:00080000:3.0:1789762557.443803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762557.443806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762557.443810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762557.443824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048883840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762557.443834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762557.443838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789762557.443843:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762557.443844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762557.443852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2007 00000100:00080000:3.0:1789762557.443858:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762557.443864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762557.443871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762557.443885:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762557/real 1789762557] req@ffff9811e1a20700 x1876700048883584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762557.443893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048883968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762557.443926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048883584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762557.443927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762557.443933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762557.443935:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762557.443943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2007 00000100:00080000:3.0:1789762557.443954:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762557.443958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762557.443965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789762557.443970:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:3.0:1789762557.443975:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762557.443977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048883584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762557.444006:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048884096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762557.444022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048883712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762557.444050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762557.444059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762557/real 1789762557] req@ffff9811e1a21500 x1876700048883712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762557.444068:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048883712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762557.444074:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762557.444077:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762557.444079:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048883712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762557.444083:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762557.444094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762557.444103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762557/real 1789762557] req@ffff9811e1a23480 x1876700048883840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762557.444115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048883840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762557.444124:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762557.444127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762557.444130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762557.444145:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762557.444167:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762557.444179:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762557/real 1789762557] req@ffff9811e1a23800 x1876700048883968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762557.444191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048883968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762557.444198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762557.444202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762557.444204:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048883968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762557.444210:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762557.444237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762557.444247:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762557/real 1789762557] req@ffff9811e1a23100 x1876700048884096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762557.444259:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048884096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762557.444267:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762557.444271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762557.444274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762557.699542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762558.339591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762558.723590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762559.363576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762559.747596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762560.387580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762560.771565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762561.411617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762561.795524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762562.435593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762562.563603:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762562.563616:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762562.563621:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762562.563634:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2012 00000100:00080000:2.0:1789762562.563648:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762562.563657:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762562.563706:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048884224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762562.563747:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762562.563752:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762562.563755:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762562.563760:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2012 00000100:00080000:2.0:1789762562.563764:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762562.563768:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762562.563780:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762562.563794:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048884352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762562.563808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762562.563812:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762562.563815:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762562.563819:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2012 00000100:00080000:2.0:1789762562.563822:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762562.563827:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762562.563855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762562.563869:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048884480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762562.563885:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048884224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762562.563889:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762562.563902:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762562.563905:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762562.563911:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2012 00000100:00080000:2.0:1789762562.563916:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762562.563922:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762562.563928:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762562.563944:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048884608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762562.563988:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762562.563996:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762562.564000:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762562.564008:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2012 00000800:00000100:1.0:1789762562.564016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762562.564022:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762562.564036:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762562.564049:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762562.564075:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048884736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762562.564086:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762562/real 1789762562] req@ffff9811d8639880 x1876700048884224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762562.564118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048884224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762562.564134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762562.564139:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762562.564157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048884224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762562.564168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048884352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762562.564202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762562.564219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762562/real 1789762562] req@ffff9811d8639500 x1876700048884352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762562.564234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048884352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762562.564264:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762562.564268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762562.564273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048884352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762562.564303:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762562.564327:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762562.564340:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762562/real 1789762562] req@ffff9811d8639f80 x1876700048884480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762562.564355:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048884480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762562.564365:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762562.564369:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762562.564375:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762562.564386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762562.564422:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762562.564507:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762562/real 1789762562] req@ffff9811d863a680 x1876700048884608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762562.564529:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048884608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762562.564540:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762562.564544:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762562.564548:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762562.564556:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762562.564587:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762562.564601:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762562/real 1789762562] req@ffff9811d8638a80 x1876700048884736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762562.564616:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048884736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762562.564625:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762562.564628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762562.564632:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048884736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762562.819645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762563.459784:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762563.843556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762564.483563:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762564.867557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762565.507661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762565.891523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762566.531607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762566.915569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762567.555588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762567.683578:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762567.683587:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762567.683590:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762567.683599:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2017 00000100:00080000:2.0:1789762567.683609:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762567.683615:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762567.683651:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048884864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762567.683682:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762567.683686:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762567.683688:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762567.683693:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2017 00000100:00080000:2.0:1789762567.683697:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762567.683700:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762567.683707:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762567.683719:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048884992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762567.683730:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762567.683733:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762567.683735:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762567.683740:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048884864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762567.683753:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2017 00000100:00080000:2.0:1789762567.683756:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762567.683760:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762567.683764:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762567.683772:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048885120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762567.683780:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762567.683783:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762567.683784:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762567.683786:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2017 00000100:00080000:2.0:1789762567.683789:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762567.683791:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762567.683794:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762567.683802:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048885248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762567.683810:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762567.683810:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762567.683816:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762567.683818:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762567.683822:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2017 00000100:00080000:2.0:1789762567.683824:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762567.683826:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762567.683829:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762567.683836:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048885376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762567.683866:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762567/real 1789762567] req@ffff9811d863b800 x1876700048884864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762567.683903:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048884864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762567.683928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762567.683934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762567.683938:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048884864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762567.683971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048884992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762567.683996:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762567.684011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762567/real 1789762567] req@ffff9811d863a300 x1876700048884992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762567.684025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048884992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762567.684034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762567.684038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762567.684042:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048884992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762567.684048:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762567.684066:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762567.684078:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762567/real 1789762567] req@ffff9811d863b480 x1876700048885120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762567.684092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048885120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762567.684121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762567.684126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762567.684129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762567.684136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762567.684159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762567.684171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762567/real 1789762567] req@ffff9811d863aa00 x1876700048885248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762567.684183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048885248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762567.684192:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762567.684196:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762567.684199:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762567.684206:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762567.684225:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762567.684235:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762567/real 1789762567] req@ffff9811d8638000 x1876700048885376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762567.684248:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048885376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762567.684256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762567.684259:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762567.684269:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762567.939534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762568.579601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762568.963600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762569.603675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762569.987550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762570.627613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762571.011532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762571.651564:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762572.035517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762572.675661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762572.803557:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762572.803566:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762572.803569:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762572.803579:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2022 00000100:00080000:2.0:1789762572.803588:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762572.803594:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762572.803633:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048885504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762572.803655:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762572.803658:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762572.803659:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762572.803663:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2022 00000100:00080000:2.0:1789762572.803667:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762572.803671:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762572.803681:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762572.803696:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048885632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762572.803712:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762572.803717:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789762572.803728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048885504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762572.803733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762572.803742:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2022 00000100:00080000:2.0:1789762572.803746:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762572.803749:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762572.803754:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762572.803766:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048885760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762572.803774:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762572.803777:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762572.803779:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762572.803782:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2022 00000100:00080000:2.0:1789762572.803785:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762572.803787:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762572.803790:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762572.803800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048885888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762572.803807:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762572.803809:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762572.803811:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762572.803814:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2022 00000100:00080000:2.0:1789762572.803817:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762572.803819:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762572.803823:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762572.803834:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762572.803848:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048886016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762572.803913:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762572/real 1789762572] req@ffff9811d8638700 x1876700048885504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762572.803937:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048885504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762572.803951:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762572.803955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762572.803960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048885504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762572.803989:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048885632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762572.804019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762572.804034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762572/real 1789762572] req@ffff9811d8638e00 x1876700048885632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762572.804048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048885632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762572.804057:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762572.804061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762572.804064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048885632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762572.804087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762572.804112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762572.804126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762572/real 1789762572] req@ffff9811d8638380 x1876700048885760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762572.804140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048885760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762572.804149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762572.804153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762572.804157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762572.804164:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762572.804186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762572.804199:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762572/real 1789762572] req@ffff9811d8638000 x1876700048885888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762572.804211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048885888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762572.804220:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762572.804224:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762572.804227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048885888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762572.804234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762572.804257:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762572.804269:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762572/real 1789762572] req@ffff9811d863aa00 x1876700048886016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762572.804290:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048886016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762572.804299:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762572.804303:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762572.804307:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762573.059556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762573.699645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762574.083594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762574.723627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762575.107575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762575.747714:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762576.131590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762576.771663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762577.155563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762577.795643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762577.923599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762577.923609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762577.923613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762577.923625:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2027 00000100:00080000:3.0:1789762577.923638:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762577.923647:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762577.923699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048886144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762577.923750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762577.923753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762577.923754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762577.923757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2027 00000100:00080000:3.0:1789762577.923760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762577.923764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762577.923773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762577.923791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048886272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762577.923799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762577.923803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762577.923805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762577.923807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2027 00000100:00080000:3.0:1789762577.923810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762577.923813:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762577.923817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762577.923825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048886400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762577.923844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762577.923848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762577.923849:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762577.923853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2027 00000100:00080000:3.0:1789762577.923864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762577.923867:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048886144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762577.923877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762577.923884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762577.923897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048886528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762577.923907:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762577.923910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762577.923912:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762577.923914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2027 00000100:00080000:3.0:1789762577.923917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762577.923919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762577.923923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762577.923930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048886656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762577.923952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762577.923996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762577/real 1789762577] req@ffff9811e1a22300 x1876700048886144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762577.924011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048886144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762577.924020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762577.924023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762577.924026:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048886144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762577.924032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048886272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762577.924049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762577.924056:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762577/real 1789762577] req@ffff9811e1a21180 x1876700048886272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762577.924083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048886272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762577.924089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762577.924091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762577.924093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048886272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762577.924097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762577.924111:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762577.924117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762577/real 1789762577] req@ffff9811e1a22680 x1876700048886400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762577.924125:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048886400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762577.924130:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762577.924132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762577.924133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762577.924153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762577.924165:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762577.924171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762577/real 1789762577] req@ffff9811e1a21c00 x1876700048886528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762577.924178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048886528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762577.924194:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762577.924196:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762577.924199:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762577.924203:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762577.924215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762577.924221:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762577/real 1789762577] req@ffff9811e1a21880 x1876700048886656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762577.924229:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048886656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762577.924233:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762577.924235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762577.924237:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048886656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762578.179573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762578.819636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762579.203559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762579.843596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762580.227597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762580.867711:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762581.251537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762581.891596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762582.275598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762582.915665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762583.043613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762583.043623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762583.043626:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762583.043637:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2032 00000100:00080000:1.0:1789762583.043649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762583.043656:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762583.043705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048886784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762583.043733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762583.043736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762583.043738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762583.043742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2032 00000100:00080000:1.0:1789762583.043749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762583.043755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762583.043764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762583.043781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048886912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762583.043787:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048886784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762583.043811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762583.043814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762583.043816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762583.043821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2032 00000100:00080000:1.0:1789762583.043827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762583.043830:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762583.043835:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762583.043848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048887040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762583.043859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762583.043863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762583.043865:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762583.043876:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762583.043877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2032 00000100:00080000:1.0:1789762583.043886:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762583.043890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762583.043897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762583.043908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048887168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762583.043915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762583.043918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762583.043920:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762583.043925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2032 00000100:00080000:1.0:1789762583.043927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762583.043929:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762583.043932:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762583.043940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048887296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762583.044140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762583/real 1789762583] req@ffff9811c92c4000 x1876700048886784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762583.044214:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048886784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762583.044228:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762583.044233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762583.044239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048886784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762583.044270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048886912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762583.044298:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762583.044322:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762583/real 1789762583] req@ffff9811c92c6680 x1876700048886912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762583.044336:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048886912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762583.044346:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762583.044350:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762583.044354:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048886912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762583.044361:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762583.044382:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762583.044394:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762583/real 1789762583] req@ffff9811c92c5c00 x1876700048887040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762583.044406:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048887040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762583.044415:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762583.044418:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762583.044422:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762583.044428:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762583.044448:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762583.044458:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762583/real 1789762583] req@ffff9811c92c7b80 x1876700048887168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762583.044512:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048887168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762583.044523:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762583.044529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762583.044535:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762583.044552:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762583.044586:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762583.044599:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762583/real 1789762583] req@ffff9811c92c5880 x1876700048887296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762583.044611:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048887296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762583.044620:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762583.044623:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762583.044627:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762583.299532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762583.939678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762584.323546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762584.963595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762585.347534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762585.987622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762586.371583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762587.011645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762587.395544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762588.035652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762588.163564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762588.163582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762588.163586:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762588.163598:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2037 00000100:00080000:3.0:1789762588.163609:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762588.163620:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762588.163664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048887424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762588.163710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762588.163713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762588.163714:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762588.163717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2037 00000100:00080000:3.0:1789762588.163720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762588.163723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762588.163732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762588.163743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048887552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762588.163750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048887424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762588.163769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762588.163773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762588.163774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762588.163778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2037 00000100:00080000:3.0:1789762588.163781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762588.163784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762588.163790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762588.163804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048887680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762588.163816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762588.163819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762588.163821:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762588.163825:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2037 00000100:00080000:3.0:1789762588.163837:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762588.163839:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762588.163846:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762588.163853:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762588.163864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048887808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762588.163871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762588.163873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762588.163874:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762588.163878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2037 00000100:00080000:3.0:1789762588.163881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762588.163884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762588.163887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762588.163888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762588/real 1789762588] req@ffff9811e1a22d80 x1876700048887424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762588.163899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048887936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762588.163937:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048887424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762588.163946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762588.163950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762588.163953:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048887424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762588.163976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048887552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762588.163994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762588.164003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762588/real 1789762588] req@ffff9811e1a22a00 x1876700048887552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762588.164011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048887552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762588.164016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762588.164018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762588.164020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048887552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762588.164024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762588.164034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762588.164040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762588/real 1789762588] req@ffff9811e1a21880 x1876700048887680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762588.164048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048887680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762588.164061:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762588.164064:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762588.164066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762588.164070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762588.164092:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762588.164099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762588/real 1789762588] req@ffff9811e1a21c00 x1876700048887808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762588.164107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048887808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762588.164111:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762588.164113:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762588.164115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762588.164118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762588.164128:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762588.164134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762588/real 1789762588] req@ffff9811e1a22680 x1876700048887936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762588.164140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048887936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762588.164145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762588.164147:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762588.164154:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048887936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762588.419562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762589.059588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762589.443545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762590.083648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762590.467535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762591.107565:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762591.491537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762592.131763:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762592.515585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762593.155713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762593.283581:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762593.283590:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762593.283593:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762593.283618:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2042 00000100:00080000:1.0:1789762593.283637:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762593.283644:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762593.283679:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048888064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762593.283705:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762593.283709:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762593.283710:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762593.283714:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2042 00000100:00080000:1.0:1789762593.283731:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762593.283735:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762593.283760:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762593.283773:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048888192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762593.283775:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048888064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762593.283791:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762593.283794:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762593.283796:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762593.283799:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2042 00000100:00080000:1.0:1789762593.283815:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762593.283819:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762593.283823:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762593.283832:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048888320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762593.283840:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762593.283842:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762593.283844:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762593.283846:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2042 00000100:00080000:1.0:1789762593.283849:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762593.283852:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762593.283856:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762593.283870:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048888448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762593.283878:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762593.283879:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048888192:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789762593.283885:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762593.283888:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762593.283892:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2042 00000100:00080000:1.0:1789762593.283896:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762593.283899:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762593.283903:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762593.283912:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762593.283917:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048888576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762593.283937:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762593.283965:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762593.539554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762594.179595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762594.563519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762595.203631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762595.587544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762596.227618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762596.355689:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789762596.355739:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762596.355753:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789762596.355784:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789762596.355791:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789762596.355797:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789762596.355805:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789762596.355845:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762593/real 1789762596] req@ffff9811c92c7b80 x1876700048888576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762596.355874:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048888576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762596.355889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762596.355895:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762596.355900:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762596.355910:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762593/real 1789762596] req@ffff9811c92c5880 x1876700048888448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762596.355926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048888448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762596.355937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762596.355941:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762596.355945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762596.355953:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762593/real 1789762596] req@ffff9811c92c6d80 x1876700048888192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762596.355968:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048888192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762596.355978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762596.355983:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762596.355987:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048888192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762596.356016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762593/real 1789762596] req@ffff9811c92c5180 x1876700048888320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762596.356032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048888320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762596.356042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762596.356046:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762596.356050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762596.356057:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762593/real 1789762596] req@ffff9811c92c5500 x1876700048888064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762596.356070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048888064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762596.356080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762596.356083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762596.356086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048888064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762596.611515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762597.251611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762597.635587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762598.275600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762598.403543:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762598.403553:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762598.403556:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762598.403565:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2047 00000100:00080000:1.0:1789762598.403574:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762598.403581:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762598.403611:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048888704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762598.403634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762598.403637:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762598.403638:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762598.403641:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2047 00000100:00080000:1.0:1789762598.403643:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762598.403645:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762598.403652:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762598.403661:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048888832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762598.403689:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762598.403692:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762598.403694:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762598.403697:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2047 00000100:00080000:1.0:1789762598.403700:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762598.403703:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762598.403718:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048888704:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762598.403723:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762598.403748:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048888960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762598.403762:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762598.403766:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762598.403768:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762598.403774:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2047 00000100:00080000:1.0:1789762598.403779:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762598.403783:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762598.403790:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762598.403804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048889088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762598.403813:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762598.403825:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762598.403831:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762598.403834:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762598.403841:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2047 00000100:00080000:1.0:1789762598.403854:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762598.403858:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762598.403865:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762598.403876:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048889216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762598.403886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762598/real 1789762598] req@ffff9811c92c5c00 x1876700048888704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762598.403917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048888704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762598.403953:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762598.403959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762598.403963:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048888704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762598.403973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048888832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762598.404006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762598.404022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762598/real 1789762598] req@ffff9811c92c6680 x1876700048888832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762598.404035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048888832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762598.404045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762598.404048:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762598.404051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048888832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762598.404087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762598.404110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762598.404122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762598/real 1789762598] req@ffff9811c92c4000 x1876700048888960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762598.404133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048888960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762598.404141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762598.404151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762598.404155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048888960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762598.404170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762598.404195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762598.404208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762598/real 1789762598] req@ffff9811c92c4e00 x1876700048889088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762598.404221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048889088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762598.404228:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762598.404232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762598.404235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762598.404255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762598.404278:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762598.404398:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762598/real 1789762598] req@ffff9811c92c7480 x1876700048889216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762598.404412:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048889216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762598.404419:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762598.404422:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762598.404425:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762598.659576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762599.299655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762599.683581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762600.323629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762600.707562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762601.347565:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762601.731516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762602.371596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762602.755520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762603.395615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762603.523544:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762603.523553:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762603.523573:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762603.523584:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2053 00000100:00080000:1.0:1789762603.523595:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762603.523602:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762603.523639:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048889344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762603.523683:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762603.523686:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762603.523687:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762603.523691:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2053 00000100:00080000:1.0:1789762603.523694:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762603.523698:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762603.523705:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762603.523716:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048889472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762603.523726:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762603.523730:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762603.523731:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762603.523735:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2053 00000100:00080000:1.0:1789762603.523742:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762603.523744:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048889344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762603.523747:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762603.523752:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762603.523757:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048889600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762603.523784:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762603.523788:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762603.523789:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762603.523795:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2053 00000100:00080000:1.0:1789762603.523798:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762603.523801:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762603.523807:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762603.523816:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048889728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762603.523824:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762603.523826:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762603.523827:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762603.523830:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2053 00000100:00080000:1.0:1789762603.523835:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762603.523838:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762603.523842:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762603.523849:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762603.523854:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048889856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762603.523904:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762603/real 1789762603] req@ffff9811c92c4380 x1876700048889344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762603.523922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048889344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762603.523931:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762603.523934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762603.523937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048889344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762603.523963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048889472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762603.523979:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762603.523988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762603/real 1789762603] req@ffff9811c92c4a80 x1876700048889472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762603.523995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048889472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762603.524001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762603.524003:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762603.524005:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048889472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762603.524009:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762603.524020:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762603.524026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762603/real 1789762603] req@ffff9811c92c5f80 x1876700048889600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762603.524043:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048889600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762603.524049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762603.524051:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762603.524053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762603.524057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762603.524071:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762603.524077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762603/real 1789762603] req@ffff9811c92c6a00 x1876700048889728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762603.524084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048889728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762603.524089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762603.524091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762603.524093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762603.524096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762603.524106:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762603.524112:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762603/real 1789762603] req@ffff9811c92c6300 x1876700048889856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762603.524118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048889856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762603.524141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762603.524144:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762603.524146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048889856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762603.779544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762604.419583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762604.803587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762605.443669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762605.827566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762606.467650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762606.851529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762607.491685:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762607.875548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762608.515623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762608.643600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762608.643621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762608.643626:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762608.643637:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2058 00000100:00080000:3.0:1789762608.643649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762608.643662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762608.643703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048889984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762608.643732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762608.643736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762608.643737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762608.643741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2058 00000100:00080000:3.0:1789762608.643744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762608.643748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762608.643757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762608.643773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048890112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762608.643777:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048889984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762608.643820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762608.643824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762608.643826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762608.643831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2058 00000100:00080000:3.0:1789762608.643837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762608.643843:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762608.643845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762608.643854:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762608.643870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048890240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762608.643881:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762608.643884:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762608/real 1789762608] req@ffff9811e1a21180 x1876700048889984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762608.643888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762608.643894:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762608.643900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2058 00000100:00080000:3.0:1789762608.643905:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762608.643915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762608.643923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762608.643926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048889984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762608.643937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762608.643942:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762608.643943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048890368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762608.643947:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048889984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762608.643962:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762608.643966:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762608.643969:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762608.643971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048890112:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789762608.643978:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2058 00000100:00080000:3.0:1789762608.643993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762608.643996:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762608.644004:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762608.644011:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762608/real 1789762608] req@ffff9811e1a22300 x1876700048890112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762608.644011:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762608.644031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048890112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762608.644032:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048890496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762608.644051:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762608.644059:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762608.644066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048890112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762608.644074:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762608.644098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762608.644111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762608/real 1789762608] req@ffff9811e1a23100 x1876700048890240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762608.644124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048890240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762608.644147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762608.644152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762608.644156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762608.644162:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762608.644186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762608.644198:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762608/real 1789762608] req@ffff9811e1a23800 x1876700048890368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762608.644211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048890368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762608.644220:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762608.644224:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762608.644228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762608.644245:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762608.644264:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762608.644274:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762608/real 1789762608] req@ffff9811e1a23480 x1876700048890496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762608.644287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048890496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762608.644296:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762608.644300:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762608.644310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762608.899522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762609.539622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762609.923550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762610.563637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762610.947622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762611.587653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762611.971639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762612.611631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762612.995544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762613.635599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762613.763572:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762613.763586:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762613.763591:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762613.763604:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2063 00000100:00080000:1.0:1789762613.763627:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762613.763636:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762613.763679:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048890624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762613.763704:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762613.763707:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762613.763709:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762613.763712:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2063 00000100:00080000:1.0:1789762613.763714:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762613.763717:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762613.763727:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762613.763737:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048890752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762613.763746:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048890624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762613.763748:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762613.763756:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762613.763759:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762613.763763:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2063 00000100:00080000:1.0:1789762613.763765:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762613.763786:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762613.763790:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762613.763800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048890880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762613.763807:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762613.763809:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762613.763810:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762613.763813:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2063 00000100:00080000:1.0:1789762613.763819:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762613.763821:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762613.763824:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762613.763831:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762613.763836:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048891008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762613.763850:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762613.763853:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762613.763854:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762613.763858:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2063 00000100:00080000:1.0:1789762613.763860:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762613.763862:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762613.763865:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762613.763874:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048891136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762613.763882:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762613/real 1789762613] req@ffff9811c92c4700 x1876700048890624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762613.763900:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048890624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762613.763909:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762613.763912:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762613.763915:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048890624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762613.763939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048890752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762613.763958:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762613.763971:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762613/real 1789762613] req@ffff9811c92c7800 x1876700048890752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762613.763985:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048890752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762613.764008:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762613.764012:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762613.764015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048890752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762613.764019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762613.764037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762613.764045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762613/real 1789762613] req@ffff9811c92c7100 x1876700048890880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762613.764053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048890880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762613.764059:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762613.764061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762613.764063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048890880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762613.764066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762613.764078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762613.764083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762613/real 1789762613] req@ffff9811c92c6300 x1876700048891008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762613.764090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048891008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762613.764095:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762613.764096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762613.764104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762613.764108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762613.764121:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762613.764128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762613/real 1789762613] req@ffff9811c92c6a00 x1876700048891136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762613.764135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048891136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762613.764148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762613.764150:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762613.764152:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762614.019533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762614.659634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762615.043600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762615.683683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762616.067590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762616.707623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762617.091600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762617.731688:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762618.115557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762618.755650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762618.883564:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762618.883582:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762618.883585:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762618.883594:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2068 00000100:00080000:1.0:1789762618.883611:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762618.883617:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762618.883648:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048891264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762618.883667:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762618.883670:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762618.883672:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762618.883675:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2068 00000100:00080000:1.0:1789762618.883677:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762618.883680:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762618.883687:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762618.883697:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048891264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762618.883699:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048891392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762618.883711:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762618.883714:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762618.883715:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762618.883718:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2068 00000100:00080000:1.0:1789762618.883720:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762618.883722:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762618.883726:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762618.883734:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048891520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762618.883746:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762618.883749:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762618.883750:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762618.883753:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2068 00000800:00000100:3.0:1789762618.883758:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762618.883760:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762618.883765:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762618.883770:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762618.883779:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048891648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762618.883785:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762618.883788:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762618.883790:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762618.883794:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2068 00000100:00100000:3.0:1789762618.883795:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762618/real 1789762618] req@ffff9811c92c5f80 x1876700048891264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762618.883797:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762618.883803:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762618.883810:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762618.883820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048891776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762618.883823:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048891264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762618.883843:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762618.883846:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762618.883850:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048891264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762618.883871:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048891392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762618.883902:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762618.883930:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762618/real 1789762618] req@ffff9811c92c4a80 x1876700048891392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762618.883946:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048891392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762618.883957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762618.883961:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762618.883965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048891392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762618.883973:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762618.883999:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762618.884012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762618/real 1789762618] req@ffff9811c92c4380 x1876700048891520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762618.884021:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048891520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762618.884027:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762618.884029:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762618.884032:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762618.884037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762618.884050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762618.884057:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762618/real 1789762618] req@ffff9811c92c7480 x1876700048891648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762618.884142:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048891648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762618.884150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762618.884152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762618.884156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762618.884161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762618.884178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762618.884186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762618/real 1789762618] req@ffff9811c92c4e00 x1876700048891776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762618.884194:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048891776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762618.884200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762618.884201:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762618.884203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048891776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762619.139609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762619.779652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762620.163571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762620.803644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762621.187611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762621.827726:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762622.211559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762622.851720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762623.235584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762623.875679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762624.003590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762624.003600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762624.003605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762624.003616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2073 00000100:00080000:2.0:1789762624.003628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762624.003636:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762624.003682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048891904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762624.003712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762624.003734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762624.003736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762624.003739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2073 00000100:00080000:2.0:1789762624.003743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762624.003747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762624.003757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762624.003768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048892032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762624.003773:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048891904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762624.003786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762624.003790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762624.003792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762624.003795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2073 00000100:00080000:2.0:1789762624.003798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762624.003801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762624.003805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762624.003814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048892160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762624.003821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762624.003824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762624.003826:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762624.003828:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2073 00000100:00080000:2.0:1789762624.003843:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762624.003846:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789762624.003851:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762624.003892:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762624.003909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048892288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762624.003920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762624.003923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762624.003925:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762624.003929:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2073 00000100:00080000:2.0:1789762624.003934:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762624.003936:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762624/real 1789762624] req@ffff9811d863b480 x1876700048891904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762624.003940:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762624.003948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762624.003961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048892416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762624.003969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048891904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762624.003994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762624.003998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762624.004002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048891904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762624.004025:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048892032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762624.004045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762624.004054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762624/real 1789762624] req@ffff9811d863a300 x1876700048892032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762624.004062:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048892032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762624.004068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762624.004070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762624.004072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048892032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762624.004076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762624.004086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762624.004092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762624/real 1789762624] req@ffff9811d863b800 x1876700048892160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762624.004100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048892160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762624.004105:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762624.004106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762624.004108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762624.004117:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762624.004133:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762624.004139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762624/real 1789762624] req@ffff9811d8638a80 x1876700048892288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762624.004147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048892288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762624.004152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762624.004154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762624.004156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762624.004159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762624.004169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762624.004175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762624/real 1789762624] req@ffff9811d863a680 x1876700048892416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762624.004182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048892416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762624.004187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762624.004188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762624.004190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762624.259553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762624.899633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762625.283599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762625.923615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762626.307578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762626.947654:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762627.331583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762627.971633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762628.355545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762628.995635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762629.123590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762629.123601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762629.123606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762629.123617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2078 00000100:00080000:2.0:1789762629.123630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762629.123638:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762629.123682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048892544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762629.123734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762629.123739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762629.123741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762629.123746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2078 00000100:00080000:2.0:1789762629.123752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762629.123758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762629.123769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762629.123781:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048892544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762629.123792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048892672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762629.123805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762629.123813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762629.123814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762629.123818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2078 00000100:00080000:2.0:1789762629.123822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762629.123827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762629.123831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762629.123852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048892800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762629.123861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762629.123864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762629.123866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762629.123869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2078 00000100:00080000:2.0:1789762629.123889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762629.123893:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762629.123898:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762629.123919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048892928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762629.123925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762629.123927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762629.123928:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762629.123930:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2078 00000100:00080000:2.0:1789762629.123932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762629.123944:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762629.123948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762629.123956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762629.123957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048893056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762629.124079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762629/real 1789762629] req@ffff9811d8639f80 x1876700048892544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762629.124108:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048892544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762629.124120:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762629.124124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762629.124128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048892544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762629.124154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048892672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762629.124181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762629.124193:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762629/real 1789762629] req@ffff9811d8639500 x1876700048892672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762629.124204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048892672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762629.124212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762629.124215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762629.124218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048892672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762629.124224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762629.124238:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762629.124246:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762629/real 1789762629] req@ffff9811d8639880 x1876700048892800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762629.124271:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048892800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762629.124278:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762629.124281:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762629.124284:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762629.124290:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762629.124308:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762629.124316:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762629/real 1789762629] req@ffff9811d863ad80 x1876700048892928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762629.124326:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048892928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762629.124333:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762629.124335:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762629.124338:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048892928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762629.124343:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762629.124356:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762629.124363:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762629/real 1789762629] req@ffff9811d8639c00 x1876700048893056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762629.124373:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048893056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762629.124409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762629.124414:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762629.124419:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762629.379610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762630.019662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762630.403555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762631.043607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762631.427507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762632.067620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762632.451552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762633.091688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762633.475525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762634.115587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762634.243549:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762634.243558:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762634.243561:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762634.243570:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2083 00000100:00080000:2.0:1789762634.243580:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762634.243586:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762634.243621:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048893184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762634.243643:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762634.243660:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762634.243662:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762634.243666:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2083 00000100:00080000:2.0:1789762634.243670:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762634.243674:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762634.243682:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762634.243690:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048893184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762634.243694:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048893312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762634.243707:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762634.243712:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762634.243714:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762634.243718:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2083 00000100:00080000:2.0:1789762634.243722:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762634.243727:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762634.243733:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762634.243746:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048893440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762634.243757:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762634.243761:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762634.243766:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762634.243768:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762634.243772:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2083 00000100:00080000:2.0:1789762634.243777:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762634.243781:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762634.243791:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762634.243801:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048893568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762634.243809:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762634.243811:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762634/real 1789762634] req@ffff9811d8639180 x1876700048893184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762634.243812:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762634.243816:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762634.243825:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2083 00000100:00080000:2.0:1789762634.243832:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762634.243836:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762634.243841:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762634.243845:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048893184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762634.243853:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048893696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762634.243864:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762634.243872:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762634.243878:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048893184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762634.243900:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048893312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762634.243920:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762634.243932:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762634/real 1789762634] req@ffff9811d863bb80 x1876700048893312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762634.243945:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048893312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762634.243962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762634.243966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762634.243970:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048893312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762634.243976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762634.243993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762634.244002:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762634/real 1789762634] req@ffff9811d863b100 x1876700048893440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762634.244014:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048893440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762634.244023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762634.244025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762634.244029:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762634.244034:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762634.244049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762634.244057:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762634/real 1789762634] req@ffff9811d8639c00 x1876700048893568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762634.244070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048893568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762634.244078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762634.244081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762634.244089:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762634.244095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762634.244112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762634.244121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762634/real 1789762634] req@ffff9811d863ad80 x1876700048893696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762634.244133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048893696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762634.244141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762634.244144:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762634.244148:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048893696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762634.499568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762635.139592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762635.523542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762636.163639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762636.547615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762637.187566:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762637.571533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762638.211593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762638.595529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762639.235685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762639.363715:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762639.363731:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762639.363737:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762639.363751:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2088 00000100:00080000:1.0:1789762639.363790:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762639.363801:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762639.363865:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048893824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762639.363908:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762639.363914:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762639.363918:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762639.363924:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2088 00000100:00080000:1.0:1789762639.363930:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762639.363937:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762639.363950:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762639.363973:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048893952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762639.363993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048893824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762639.363999:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762639.364006:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762639.364014:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762639.364022:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2088 00000100:00080000:1.0:1789762639.364028:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762639.364036:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762639.364045:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762639.364068:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048894080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762639.364085:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762639.364119:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762639.364130:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762639.364135:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762639.364146:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2088 00000100:00080000:1.0:1789762639.364154:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762639.364164:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762639.364178:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762639.364180:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762639/real 1789762639] req@ffff9811c92c4000 x1876700048893824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762639.364212:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048894208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762639.364213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048893824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762639.364258:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762639.364271:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762639.364278:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762639.364278:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762639.364306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762639.364315:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2088 00000100:00100000:3.0:1789762639.364316:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048893824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762639.364332:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762639.364342:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762639.364352:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762639.364362:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048893952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762639.364395:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048894336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762639.364426:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762639.364448:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762639/real 1789762639] req@ffff9811c92c6680 x1876700048893952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762639.364497:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048893952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762639.364507:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762639.364510:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762639.364512:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048893952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762639.364518:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762639.364534:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762639.364542:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762639/real 1789762639] req@ffff9811c92c5c00 x1876700048894080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762639.364549:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048894080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762639.364555:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762639.364557:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762639.364559:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762639.364577:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762639.364598:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762639.364605:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762639/real 1789762639] req@ffff9811c92c5500 x1876700048894208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762639.364613:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048894208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762639.364618:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762639.364620:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762639.364623:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762639.364645:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762639.364655:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762639.364661:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762639/real 1789762639] req@ffff9811c92c5180 x1876700048894336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762639.364668:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048894336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762639.364673:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762639.364674:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762639.364676:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762639.619554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762640.259687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762640.643538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762641.283632:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762641.667539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762642.307617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762642.691584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762643.331674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762643.715570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762644.355593:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762644.483556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762644.483568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762644.483573:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762644.483584:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2094 00000100:00080000:1.0:1789762644.483595:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762644.483603:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762644.483641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048894464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762644.483687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762644.483697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762644.483699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762644.483704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2094 00000100:00080000:1.0:1789762644.483709:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762644.483715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762644.483726:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762644.483736:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048894464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762644.483743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048894592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762644.483760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762644.483764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762644.483766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762644.483771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2094 00000100:00080000:1.0:1789762644.483775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762644.483780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762644.483785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762644.483797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048894720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762644.483808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762644.483814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762644.483819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762644.483822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762644.483829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2094 00000100:00080000:1.0:1789762644.483841:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762644.483847:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762644.483854:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762644.483857:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762644/real 1789762644] req@ffff9811c92c6d80 x1876700048894464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762644.483874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048894848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762644.483877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048894464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762644.483909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762644.483910:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762644.483918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762644.483919:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762644.483923:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762644.483928:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048894464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762644.483933:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2094 00000100:00080000:1.0:1789762644.483941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762644.483946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762644.483952:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762644.483959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048894592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762644.483968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048894976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762644.483983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762644.483996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762644/real 1789762644] req@ffff9811c92c5880 x1876700048894592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762644.484006:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048894592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762644.484012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762644.484015:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762644.484017:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048894592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762644.484028:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762644.484043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762644.484050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762644/real 1789762644] req@ffff9811c92c7b80 x1876700048894720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762644.484058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048894720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762644.484064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762644.484066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762644.484069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762644.484072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762644.484082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762644.484088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762644/real 1789762644] req@ffff9811c92c5180 x1876700048894848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762644.484095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048894848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762644.484100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762644.484102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762644.484104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762644.484115:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762644.484124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762644.484130:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762644/real 1789762644] req@ffff9811c92c5500 x1876700048894976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762644.484160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048894976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762644.484166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762644.484168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762644.484170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048894976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762644.739599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762645.379640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762645.763545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762646.403726:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762646.787606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762647.427620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762647.811539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762648.451694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762648.835529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762649.475586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762649.603591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762649.603603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762649.603608:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762649.603619:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2099 00000100:00080000:2.0:1789762649.603630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762649.603639:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762649.603681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048895104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762649.603711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762649.603731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762649.603734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762649.603741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2099 00000100:00080000:2.0:1789762649.603747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762649.603753:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048895104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762649.603756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762649.603771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762649.603787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048895232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762649.603800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762649.603803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762649.603805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762649.603808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2099 00000100:00080000:2.0:1789762649.603812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762649.603815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762649.603819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762649.603821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762649.603833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048895360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762649.603843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762649.603847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762649.603848:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762649.603852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2099 00000100:00080000:2.0:1789762649.603854:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762649.603857:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762649.603865:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762649/real 1789762649] req@ffff9811d8639880 x1876700048895104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762649.603869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762649.603885:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048895488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762649.603891:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048895104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762649.603899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762649.603909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762649.603912:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762649.603914:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762649.603917:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2099 00000100:00080000:2.0:1789762649.603922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762649.603922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789762649.603927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762649.603930:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762649.603939:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048895104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762649.603942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048895616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762649.603967:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048895232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762649.603988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762649.603998:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762649/real 1789762649] req@ffff9811d8639500 x1876700048895232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762649.604017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048895232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762649.604024:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762649.604027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762649.604030:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048895232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762649.604035:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762649.604050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762649.604058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762649/real 1789762649] req@ffff9811d8639f80 x1876700048895360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762649.604067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048895360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762649.604073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762649.604075:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762649.604078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762649.604082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762649.604094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762649.604100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762649/real 1789762649] req@ffff9811d863a680 x1876700048895488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762649.604107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048895488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762649.604125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762649.604128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762649.604131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762649.604135:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762649.604150:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762649.604157:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762649/real 1789762649] req@ffff9811d8638a80 x1876700048895616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762649.604165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048895616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762649.604171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762649.604173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762649.604175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048895616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762649.859592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762650.499577:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762650.883550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762651.523623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762651.907583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762652.547671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762652.931531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762653.571577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762653.955546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762654.595599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762654.723574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762654.723582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762654.723590:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762654.723599:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2104 00000100:00080000:0.0:1789762654.723610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762654.723616:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762654.723647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048895744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762654.723672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762654.723685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762654.723687:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762654.723691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2104 00000100:00080000:0.0:1789762654.723695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762654.723702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762654.723704:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048895744:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762654.723711:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762654.723724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048895872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762654.723735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762654.723737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762654.723739:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762654.723741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2104 00000100:00080000:0.0:1789762654.723745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762654.723748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762654.723752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762654.723761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048896000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762654.723764:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762654.723782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762654.723786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762654.723789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762654.723794:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2104 00000100:00080000:0.0:1789762654.723799:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762654.723799:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762654/real 1789762654] req@ffff9811c2e29180 x1876700048895744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762654.723809:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762654.723820:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048895744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762654.723825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762654.723831:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762654.723835:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762654.723839:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048895744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762654.723843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048896128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762654.723853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762654.723860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762654.723862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762654.723866:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2104 00000100:00080000:0.0:1789762654.723870:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762654.723873:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762654.723877:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762654.723881:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048895872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762654.723891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048896256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762654.723907:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762654.723919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762654/real 1789762654] req@ffff9811c2e2bb80 x1876700048895872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762654.723929:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048895872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762654.723935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762654.723938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762654.723941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048895872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762654.723945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762654.723956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762654.723962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762654/real 1789762654] req@ffff9811c2e29880 x1876700048896000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762654.723970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048896000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762654.723977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762654.723980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762654.723984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762654.723990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762654.724013:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762654.724028:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762654/real 1789762654] req@ffff9811c2e2ad80 x1876700048896128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762654.724048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048896128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762654.724055:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762654.724058:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762654.724062:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762654.724083:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762654.724100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762654.724109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762654/real 1789762654] req@ffff9811c2e2b100 x1876700048896256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762654.724117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048896256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762654.724123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762654.724126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762654.724128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762654.979544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762655.619580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762656.003564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762656.643625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762657.027549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762657.667575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762658.051591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762658.691641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762659.075569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762659.715557:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762659.843608:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762659.843622:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762659.843627:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762659.843638:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2109 00000100:00080000:1.0:1789762659.843650:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762659.843659:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762659.843697:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048896384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762659.843752:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762659.843755:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762659.843757:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762659.843761:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2109 00000100:00080000:1.0:1789762659.843765:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762659.843769:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762659.843779:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762659.843794:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048896512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762659.843807:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762659.843812:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762659.843815:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762659.843820:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2109 00000100:00080000:1.0:1789762659.843825:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762659.843832:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762659.843839:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762659.843851:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048896384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762659.843853:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048896640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762659.843868:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762659.843871:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762659.843873:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762659.843877:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2109 00000100:00080000:1.0:1789762659.843887:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762659.843892:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762659.843898:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762659.843909:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048896768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762659.843918:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762659.843921:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762659.843922:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762659.843925:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2109 00000100:00080000:1.0:1789762659.843927:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762659.843930:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762659.843935:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762659.843945:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048896896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762659.843967:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048896512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762659.844019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762659.844039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762659.844055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762660.099595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762660.739581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762661.123578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762661.763581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762662.147536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762662.787561:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762662.915684:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762662.915721:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762662.915749:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762662.915789:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762662.915799:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762662.915810:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762662.915819:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762662.915840:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762659/real 1789762662] req@ffff9811c92c4000 x1876700048896640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762662.915870:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048896640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762662.915883:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762662.915889:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762662.915893:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762662.915903:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762659/real 1789762662] req@ffff9811c92c4e00 x1876700048896768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762662.915917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048896768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762662.915927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762662.915930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762662.915934:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762662.915951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762659/real 1789762662] req@ffff9811c92c7480 x1876700048896896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762662.915961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048896896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762662.915967:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762662.915970:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762662.915972:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048896896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762662.915976:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762659/real 1789762662] req@ffff9811c92c5c00 x1876700048896384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762662.915983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048896384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762662.915988:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762662.915990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762662.915992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048896384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762662.915999:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762659/real 1789762662] req@ffff9811c92c6680 x1876700048896512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762662.916012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048896512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762662.916021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762662.916025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762662.916029:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048896512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762663.171544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762663.811613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762664.195535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762664.835575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762664.963560:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762664.963568:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762664.963572:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762664.963584:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2114 00000100:00080000:1.0:1789762664.963595:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762664.963603:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762664.963640:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048897024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762664.963665:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762664.963680:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762664.963681:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762664.963685:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2114 00000100:00080000:1.0:1789762664.963695:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762664.963700:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762664.963710:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762664.963724:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048897152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762664.963737:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762664.963742:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762664.963745:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762664.963749:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048897024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762664.963753:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2114 00000100:00080000:1.0:1789762664.963760:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762664.963763:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762664.963767:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762664.963780:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048897280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762664.963792:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762664.963796:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762664.963798:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762664.963802:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2114 00000100:00080000:1.0:1789762664.963807:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762664.963813:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762664.963827:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762664.963841:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762664.963856:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048897408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762664.963861:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762664/real 1789762664] req@ffff9811c92c4380 x1876700048897024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762664.963876:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762664.963882:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762664.963885:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762664.963890:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2114 00000100:00100000:2.0:1789762664.963895:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048897024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762664.963898:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762664.963907:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762664.963915:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762664.963927:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048897536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762664.963929:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762664.963941:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762664.963949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048897024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762664.963958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048897152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762664.963987:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762664.964000:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762664/real 1789762664] req@ffff9811c92c4a80 x1876700048897152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762664.964013:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048897152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762664.964023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762664.964027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762664.964030:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048897152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762664.964055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762664.964088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762664.964100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762664/real 1789762664] req@ffff9811c92c5f80 x1876700048897280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762664.964113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048897280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762664.964122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762664.964126:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762664.964129:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762664.964136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762664.964157:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762664.964169:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762664/real 1789762664] req@ffff9811c92c6a00 x1876700048897408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762664.964199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048897408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762664.964208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762664.964213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762664.964216:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762664.964223:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762664.964246:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762664.964264:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762664/real 1789762664] req@ffff9811c92c6300 x1876700048897536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762664.964293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048897536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762664.964302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762664.964306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762664.964312:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762665.219548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762665.859606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762666.243530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762666.883698:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762667.267530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762667.907650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762668.291525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762668.931668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762669.315521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762669.955629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762670.083608:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762670.083620:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762670.083624:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762670.083636:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2119 00000100:00080000:1.0:1789762670.083652:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762670.083661:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762670.083713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048897664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762670.083738:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762670.083741:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762670.083766:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762670.083769:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2119 00000100:00080000:1.0:1789762670.083772:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762670.083775:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762670.083786:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762670.083796:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048897792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762670.083805:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762670.083807:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762670.083808:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762670.083811:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2119 00000100:00080000:1.0:1789762670.083813:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762670.083815:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762670.083818:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762670.083824:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048897920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762670.083830:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762670.083832:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762670.083833:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762670.083835:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2119 00000100:00080000:1.0:1789762670.083837:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762670.083840:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762670.083844:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762670.083857:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048898048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762670.083866:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762670.083869:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762670.083870:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762670.083873:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2119 00000100:00080000:1.0:1789762670.083876:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762670.083878:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762670.083881:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762670.083889:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048898176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762670.083914:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048897664:192.168.203.113@tcp:250:kworker.0 00000800:00000100:2.0:1789762670.084012:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762670.084065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762670/real 1789762670] req@ffff9811c92c7100 x1876700048897664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762670.084091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048897664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762670.084106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762670.084112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762670.084118:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048897664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762670.084127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048897792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762670.084188:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762670.084207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762670/real 1789762670] req@ffff9811c92c7800 x1876700048897792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762670.084247:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048897792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762670.084260:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762670.084268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762670.084275:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048897792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762670.084287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762670.084332:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762670.084355:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762670/real 1789762670] req@ffff9811c92c4700 x1876700048897920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762670.084381:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048897920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762670.084392:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762670.084396:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762670.084400:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048897920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762670.084408:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762670.084437:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762670.084451:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762670/real 1789762670] req@ffff9811c92c6300 x1876700048898048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762670.084521:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048898048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762670.084551:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762670.084555:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762670.084560:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762670.084599:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762670.084630:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762670.084644:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762670/real 1789762670] req@ffff9811c92c6a00 x1876700048898176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762670.084659:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048898176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762670.084669:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762670.084673:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762670.084677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762670.339538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762670.979638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762671.363540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762672.003698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762672.387587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762673.027630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762673.411610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762674.051703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762674.435572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762675.075590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762675.203580:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762675.203590:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762675.203593:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762675.203602:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2124 00000100:00080000:2.0:1789762675.203611:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762675.203624:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762675.203671:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048898304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762675.203696:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762675.203699:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762675.203700:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762675.203703:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2124 00000100:00080000:2.0:1789762675.203706:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762675.203709:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762675.203716:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762675.203725:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048898432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762675.203732:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762675.203735:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762675.203737:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762675.203739:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2124 00000100:00080000:2.0:1789762675.203741:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762675.203745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048898304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762675.203746:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762675.203753:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762675.203760:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048898560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762675.203768:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762675.203771:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762675.203772:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762675.203780:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2124 00000100:00080000:2.0:1789762675.203794:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762675.203797:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762675.203802:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762675.203806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762675.203815:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048898688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762675.203824:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762675.203826:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762675.203827:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762675.203829:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2124 00000100:00080000:2.0:1789762675.203832:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762675.203834:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762675.203837:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762675.203846:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762675/real 1789762675] req@ffff9811d863b800 x1876700048898304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762675.203848:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048898816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762675.203889:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048898304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762675.203899:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762675.203903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762675.203906:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048898304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762675.203912:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048898432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762675.203927:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762675.203936:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762675/real 1789762675] req@ffff9811d863a300 x1876700048898432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762675.203957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048898432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762675.203964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762675.203966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762675.203968:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048898432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762675.203987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762675.204001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762675.204008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762675/real 1789762675] req@ffff9811d863b480 x1876700048898560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762675.204016:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048898560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762675.204021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762675.204024:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762675.204027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762675.204033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762675.204054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762675.204066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762675/real 1789762675] req@ffff9811d863aa00 x1876700048898688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762675.204078:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048898688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762675.204118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762675.204122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762675.204127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762675.204134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762675.204171:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762675.204183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762675/real 1789762675] req@ffff9811d8638000 x1876700048898816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762675.204195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048898816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762675.204203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762675.204207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762675.204210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048898816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762675.459568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762676.099599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762676.483574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762677.123667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762677.507566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762678.147709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762678.531573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762679.171665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762679.555551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762680.195597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762680.323559:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762680.323570:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762680.323574:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762680.323584:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2129 00000100:00080000:1.0:1789762680.323595:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762680.323603:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762680.323643:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048898944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762680.323671:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762680.323674:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762680.323676:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762680.323678:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2129 00000100:00080000:1.0:1789762680.323681:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762680.323698:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762680.323707:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762680.323717:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048899072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762680.323727:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762680.323730:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762680.323731:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762680.323734:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2129 00000100:00080000:1.0:1789762680.323737:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762680.323740:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762680.323743:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762680.323752:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048899200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762680.323752:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048898944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762680.323766:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762680.323769:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762680.323770:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762680.323772:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2129 00000100:00080000:1.0:1789762680.323773:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762680.323776:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762680.323779:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762680.323787:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048899328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762680.323799:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762680.323805:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762680.323807:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762680.323810:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2129 00000100:00080000:1.0:1789762680.323813:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762680.323816:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762680.323820:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762680.323829:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048899456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762680.323835:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762680.323883:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762680/real 1789762680] req@ffff9811c92c5f80 x1876700048898944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762680.323898:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048898944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762680.323907:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762680.323916:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762680.323919:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048898944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762680.323925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048899072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762680.323947:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762680.323962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762680/real 1789762680] req@ffff9811c92c4a80 x1876700048899072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762680.323985:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048899072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762680.323994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762680.323997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762680.324001:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048899072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762680.324029:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762680.324045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762680.324053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762680/real 1789762680] req@ffff9811c92c4380 x1876700048899200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762680.324061:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048899200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762680.324080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762680.324082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762680.324084:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762680.324089:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762680.324110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762680.324118:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762680/real 1789762680] req@ffff9811c92c6680 x1876700048899328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762680.324125:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048899328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762680.324131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762680.324133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762680.324135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762680.324138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762680.324148:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762680.324154:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762680/real 1789762680] req@ffff9811c92c5c00 x1876700048899456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762680.324161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048899456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762680.324166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762680.324167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762680.324169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762680.579570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762681.219616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762681.603574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762682.243668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762682.627578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762683.267609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762683.651566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762684.291599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762684.675558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762685.315673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762685.443545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762685.443557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762685.443561:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762685.443572:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2135 00000100:00080000:0.0:1789762685.443584:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762685.443592:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762685.443626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048899584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762685.443662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762685.443666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762685.443668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762685.443674:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2135 00000100:00080000:0.0:1789762685.443679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762685.443685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762685.443690:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048899584:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762685.443697:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762685.443716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048899712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762685.443727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762685.443730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762685.443731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762685.443734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2135 00000100:00080000:0.0:1789762685.443738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762685.443741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762685.443746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762685.443756:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762685.443761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048899840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762685.443776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762685.443780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762685.443783:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762685.443787:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2135 00000100:00100000:3.0:1789762685.443799:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762685/real 1789762685] req@ffff9811c2e2aa00 x1876700048899584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762685.443801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762685.443810:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762685.443817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762685.443823:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048899584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762685.443832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048899968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762685.443843:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762685.443845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762685.443851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762685.443854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762685.443855:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762685.443859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2135 00000100:00080000:0.0:1789762685.443863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762685.443865:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048899584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762685.443868:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762685.443875:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762685.443883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048900096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762685.443899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048899712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762685.443923:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762685.443933:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762685/real 1789762685] req@ffff9811c2e28380 x1876700048899712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762685.443942:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048899712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762685.443949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762685.443951:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762685.443954:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048899712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762685.443958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762685.443981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762685.443989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762685/real 1789762685] req@ffff9811c2e28a80 x1876700048899840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762685.443997:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048899840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762685.444012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762685.444015:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762685.444018:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762685.444022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762685.444055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762685.444064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762685/real 1789762685] req@ffff9811c2e29f80 x1876700048899968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762685.444072:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048899968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762685.444078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762685.444080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762685.444083:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048899968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762685.444086:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762685.444099:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762685.444104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762685/real 1789762685] req@ffff9811c2e2b800 x1876700048900096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762685.444112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048900096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762685.444116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762685.444118:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762685.444125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762685.699599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762686.339714:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762686.723580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762687.363690:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762687.747666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762688.387672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762688.771539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762689.411586:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762689.795573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762690.435617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762690.563589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762690.563600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762690.563605:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762690.563616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2140 00000100:00080000:0.0:1789762690.563628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762690.563637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762690.563675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048900224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762690.563721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762690.563726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762690.563729:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762690.563735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2140 00000100:00080000:0.0:1789762690.563741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762690.563747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762690.563757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762690.563766:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048900224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762690.563776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048900352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762690.563792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762690.563797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762690.563799:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762690.563804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2140 00000100:00080000:0.0:1789762690.563822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762690.563828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762690.563833:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762690.563837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762690.563855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048900480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762690.563867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762690.563871:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789762690.563873:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762690/real 1789762690] req@ffff9811c2e2b480 x1876700048900224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762690.563875:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762690.563886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2140 00000100:00100000:1.0:1789762690.563897:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048900224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762690.563901:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762690.563911:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762690.563912:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789762690.563916:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762690.563922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048900224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762690.563924:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762690.563950:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048900352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762690.563959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048900608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762690.563975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762690.563976:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762690.563983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762690.563986:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762690.564006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762690/real 1789762690] req@ffff9811c2e29500 x1876700048900352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762690.564008:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2140 00000100:00080000:0.0:1789762690.564017:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762690.564025:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762690.564031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048900352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762690.564032:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762690.564051:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048900736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762690.564054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762690.564064:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762690.564072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048900352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762690.564081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762690.564109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762690.564123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762690/real 1789762690] req@ffff9811c2e28000 x1876700048900480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762690.564136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048900480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762690.564144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762690.564148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762690.564151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762690.564167:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762690.564189:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762690.564200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762690/real 1789762690] req@ffff9811c2e2a300 x1876700048900608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762690.564213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048900608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762690.564222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762690.564224:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762690.564234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762690.564239:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762690.564254:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762690.564261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762690/real 1789762690] req@ffff9811c2e2b480 x1876700048900736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762690.564269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048900736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762690.564274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762690.564276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762690.564279:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048900736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762690.819568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762691.459580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762691.843535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762692.483629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762692.867558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762693.507621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762693.891530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762694.531689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762694.915561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762695.555570:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762695.683606:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762695.683618:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762695.683623:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762695.683633:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2145 00000100:00080000:3.0:1789762695.683644:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762695.683652:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762695.683689:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048900864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762695.683718:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762695.683722:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762695.683725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762695.683729:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2145 00000100:00080000:3.0:1789762695.683733:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762695.683737:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762695.683745:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762695.683749:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048900864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762695.683759:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048900992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762695.683769:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762695.683772:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762695.683774:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762695.683776:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2145 00000100:00080000:3.0:1789762695.683795:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762695.683798:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762695.683803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762695.683807:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762695.683816:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048901120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762695.683825:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762695.683828:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762695.683829:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762695.683832:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2145 00000100:00080000:3.0:1789762695.683835:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762695.683838:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762695.683842:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762695.683846:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762695/real 1789762695] req@ffff9811e1a21500 x1876700048900864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762695.683853:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048901248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762695.683862:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762695.683866:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048900864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762695.683867:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762695.683870:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762695.683876:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2145 00000100:00080000:3.0:1789762695.683881:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762695.683887:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762695.683891:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762695.683892:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762695.683898:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048901376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762695.683901:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762695.683911:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048900864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762695.683941:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048900992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762695.683977:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762695.684002:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762695/real 1789762695] req@ffff9811e1a20700 x1876700048900992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762695.684012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048900992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762695.684018:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762695.684020:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762695.684023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048900992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762695.684027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762695.684041:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762695.684048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762695/real 1789762695] req@ffff9811e1a20380 x1876700048901120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762695.684056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048901120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762695.684061:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762695.684063:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762695.684064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762695.684068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762695.684078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762695.684084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762695/real 1789762695] req@ffff9811e1a20e00 x1876700048901248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762695.684091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048901248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762695.684101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762695.684104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762695.684106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762695.684109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762695.684123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762695.684129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762695/real 1789762695] req@ffff9811e1a20000 x1876700048901376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762695.684136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048901376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762695.684141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762695.684143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762695.684145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762695.939562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762696.579559:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762696.963570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762697.603645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762697.987622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762698.627620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762699.011528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762699.651584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762700.035562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762700.675595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762700.803579:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762700.803591:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762700.803596:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762700.803608:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2150 00000100:00080000:1.0:1789762700.803622:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762700.803630:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762700.803682:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048901504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762700.803716:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762700.803738:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762700.803741:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762700.803746:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2150 00000100:00080000:1.0:1789762700.803751:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762700.803756:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762700.803769:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762700.803788:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048901632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762700.803788:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048901504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762700.803808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762700.803812:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762700.803814:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762700.803819:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2150 00000100:00080000:1.0:1789762700.803823:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762700.803828:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762700.803834:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762700.803849:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048901760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762700.803860:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762700.803865:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762700.803868:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762700.803871:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2150 00000100:00080000:1.0:1789762700.803876:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762700.803877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762700.803885:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762700.803899:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762700.803929:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048901888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762700.803941:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762700.803943:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762700/real 1789762700] req@ffff9811c92c7480 x1876700048901504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762700.803947:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762700.803952:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762700.803960:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2150 00000100:00080000:1.0:1789762700.803966:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762700.803967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048901504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762700.803975:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789762700.803981:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762700.803983:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762700.803986:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762700.803992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048901504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762700.804002:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048902016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762700.804018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048901632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762700.804040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762700.804050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762700/real 1789762700] req@ffff9811c92c4e00 x1876700048901632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762700.804059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048901632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762700.804064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762700.804067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762700.804069:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048901632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762700.804072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762700.804084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762700.804101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762700/real 1789762700] req@ffff9811c92c4000 x1876700048901760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762700.804109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048901760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762700.804122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762700.804124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762700.804126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762700.804130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762700.804144:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762700.804151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762700/real 1789762700] req@ffff9811c92c5500 x1876700048901888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762700.804159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048901888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762700.804164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762700.804166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762700.804168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048901888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762700.804171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762700.804181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762700.804186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762700/real 1789762700] req@ffff9811c92c5180 x1876700048902016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762700.804192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048902016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762700.804202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762700.804205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762700.804208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762701.059567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762701.699675:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762702.083543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762702.723649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762703.107543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762703.747629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762704.131562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762704.771650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762705.155565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762705.795588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762705.923597:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762705.923607:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762705.923612:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762705.923623:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2155 00000100:00080000:2.0:1789762705.923634:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762705.923662:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762705.923702:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048902144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762705.923727:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762705.923730:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762705.923731:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762705.923734:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2155 00000100:00080000:2.0:1789762705.923737:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762705.923740:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762705.923749:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762705.923760:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048902272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762705.923770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762705.923773:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762705.923776:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762705.923777:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048902144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762705.923781:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2155 00000100:00080000:2.0:1789762705.923786:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762705.923789:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762705.923792:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762705.923800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048902400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762705.923806:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762705.923808:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762705.923814:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762705.923817:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2155 00000100:00080000:2.0:1789762705.923820:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762705.923823:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762705.923828:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762705.923836:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048902528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762705.923842:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762705.923845:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762705.923846:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762705.923848:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2155 00000100:00080000:2.0:1789762705.923850:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762705.923852:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762705.923855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762705.923861:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762705.923864:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048902656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762705.923928:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762705/real 1789762705] req@ffff9811d8638380 x1876700048902144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762705.923948:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048902144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762705.923957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762705.923960:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762705.923963:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048902144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762705.923987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048902272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762705.924006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762705.924016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762705/real 1789762705] req@ffff9811d8638e00 x1876700048902272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762705.924024:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048902272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762705.924029:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762705.924031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762705.924034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048902272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762705.924045:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762705.924060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762705.924067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762705/real 1789762705] req@ffff9811d8638700 x1876700048902400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762705.924075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048902400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762705.924089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762705.924091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762705.924093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762705.924097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762705.924110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762705.924116:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762705/real 1789762705] req@ffff9811d8638000 x1876700048902528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762705.924124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048902528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762705.924128:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762705.924130:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762705.924132:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762705.924136:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762705.924146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762705.924151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762705/real 1789762705] req@ffff9811d863aa00 x1876700048902656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762705.924165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048902656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762705.924170:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762705.924172:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762705.924174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048902656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762706.179608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762706.819626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762707.203530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762707.843624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762708.227557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762708.867587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762709.251550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762709.891572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762710.275548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762710.915596:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762711.043537:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762711.043546:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762711.043550:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762711.043558:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2160 00000100:00080000:2.0:1789762711.043568:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762711.043574:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762711.043614:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048902784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762711.043635:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762711.043638:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762711.043640:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762711.043643:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2160 00000100:00080000:2.0:1789762711.043646:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762711.043649:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762711.043656:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762711.043664:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048902912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762711.043681:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048902784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762711.043682:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762711.043692:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762711.043693:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762711.043696:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2160 00000100:00080000:2.0:1789762711.043699:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762711.043702:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762711.043706:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762711.043713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048903040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762711.043721:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762711.043724:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762711.043725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762711.043733:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2160 00000100:00080000:2.0:1789762711.043737:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762711.043740:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762711.043745:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762711.043751:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762711.043759:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048903168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762711.043767:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762711.043769:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762711.043771:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762711.043773:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2160 00000100:00080000:2.0:1789762711.043777:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762711.043781:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762711.043786:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762711.043793:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762711/real 1789762711] req@ffff9811d863b480 x1876700048902784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762711.043801:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048903296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762711.043824:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048902784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762711.043849:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762711.043855:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762711.043859:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048902784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762711.043882:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048902912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762711.043911:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762711.043923:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762711/real 1789762711] req@ffff9811d863a300 x1876700048902912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762711.043936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048902912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762711.043945:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762711.043948:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762711.043952:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048902912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762711.043958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762711.043974:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762711.043984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762711/real 1789762711] req@ffff9811d863b800 x1876700048903040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762711.043995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048903040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762711.044005:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762711.044008:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762711.044011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762711.044017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762711.044033:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762711.044041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762711/real 1789762711] req@ffff9811d8638a80 x1876700048903168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762711.044066:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048903168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762711.044075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762711.044078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762711.044082:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762711.044087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762711.044104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762711.044114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762711/real 1789762711] req@ffff9811d863a680 x1876700048903296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762711.044125:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048903296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762711.044134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762711.044137:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762711.044140:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762711.299522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762711.939636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762712.323558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762712.963635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762713.347532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762713.987601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762714.371530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762715.011598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762715.395516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762716.035558:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762716.163574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762716.163599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762716.163602:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762716.163611:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2165 00000100:00080000:3.0:1789762716.163619:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762716.163625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762716.163653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048903424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762716.163679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762716.163682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762716.163684:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762716.163686:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2165 00000100:00080000:3.0:1789762716.163689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762716.163691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762716.163698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762716.163707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048903552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762716.163715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762716.163719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762716.163721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762716.163724:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048903424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762716.163726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2165 00000100:00080000:3.0:1789762716.163733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762716.163735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762716.163739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762716.163748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048903680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762716.163761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762716.163763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762716.163765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762716.163768:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2165 00000100:00080000:3.0:1789762716.163773:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762716.163778:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762716.163784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762716.163787:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762716.163820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048903808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762716.163833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762716.163837:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762716.163839:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762716.163842:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762716/real 1789762716] req@ffff9811e1a21f80 x1876700048903424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762716.163847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2165 00000100:00080000:3.0:1789762716.163857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762716.163865:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048903424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762716.163866:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762716.163879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762716.163881:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762716.163886:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762716.163890:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048903424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762716.163899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048903936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762716.163913:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048903552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762716.163939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762716.163948:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762716/real 1789762716] req@ffff9811e1a20a80 x1876700048903552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762716.163957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048903552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762716.163963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762716.163965:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762716.163967:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048903552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762716.163971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762716.163982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762716.163988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762716/real 1789762716] req@ffff9811e1a20000 x1876700048903680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762716.163995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048903680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762716.164001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762716.164003:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762716.164006:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762716.164012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762716.164032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762716.164044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762716/real 1789762716] req@ffff9811e1a20e00 x1876700048903808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762716.164071:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048903808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762716.164079:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762716.164082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762716.164085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762716.164091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762716.164112:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762716.164125:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762716/real 1789762716] req@ffff9811e1a20380 x1876700048903936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762716.164138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048903936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762716.164148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762716.164151:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762716.164155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048903936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762716.419523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762717.059607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762717.443561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762718.083770:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762718.467588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762719.107603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762719.491579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762720.131739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762720.515586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762721.155642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762721.283567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762721.283578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762721.283582:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762721.283591:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2170 00000100:00080000:2.0:1789762721.283601:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762721.283609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762721.283677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048904064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762721.283709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762721.283713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762721.283715:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762721.283718:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2170 00000100:00080000:2.0:1789762721.283722:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762721.283726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762721.283734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762721.283748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048904192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762721.283780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762721.283785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762721.283788:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762721.283809:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048904064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762721.283811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2170 00000100:00080000:2.0:1789762721.283819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762721.283824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762721.283829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762721.283841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048904320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762721.283852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762721.283855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762721.283857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762721.283867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2170 00000100:00080000:2.0:1789762721.283872:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762721.283878:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762721.283884:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762721.283890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762721.283910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048904448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762721.283925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762721.283929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762721/real 1789762721] req@ffff9811d8639f80 x1876700048904064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762721.283933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762721.283938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762721.283947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2170 00000100:00080000:2.0:1789762721.283952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762721.283954:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048904064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762721.283960:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762721.283978:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762721.283991:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789762721.283997:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048904576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762721.283998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762721.284008:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048904064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762721.284024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048904192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762721.284049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762721.284059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762721/real 1789762721] req@ffff9811d8639500 x1876700048904192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762721.284069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048904192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762721.284076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762721.284078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762721.284081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048904192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762721.284102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762721.284116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762721.284123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762721/real 1789762721] req@ffff9811d8639880 x1876700048904320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762721.284133:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048904320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762721.284139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762721.284141:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762721.284143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762721.284154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762721.284171:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762721.284179:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762721/real 1789762721] req@ffff9811d863ad80 x1876700048904448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762721.284188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048904448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762721.284195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762721.284197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762721.284199:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762721.284203:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762721.284215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762721.284222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762721/real 1789762721] req@ffff9811d8639c00 x1876700048904576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762721.284231:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048904576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762721.284237:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762721.284239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762721.284241:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762721.539546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762722.179653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762722.563561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762723.203599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762723.587540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762724.227653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762724.611589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762725.251578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762725.635538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762726.275611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762726.403541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762726.403551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762726.403556:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762726.403566:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2175 00000100:00080000:2.0:1789762726.403579:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762726.403589:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762726.403638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048904704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762726.403685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762726.403688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762726.403693:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762726.403697:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2175 00000100:00080000:2.0:1789762726.403700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762726.403703:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762726.403713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762726.403740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048904832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762726.403745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048904704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762726.403756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762726.403760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762726.403761:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762726.403765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2175 00000100:00080000:2.0:1789762726.403767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762726.403770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762726.403775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762726.403784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048904960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762726.403792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762726.403795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762726.403796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762726.403799:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2175 00000100:00080000:2.0:1789762726.403814:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762726.403818:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762726.403823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762726.403834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048905088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762726.403841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762726.403843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762726.403844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762726.403847:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2175 00000100:00080000:2.0:1789762726.403849:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762726.403852:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762726.403856:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762726.403865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048905216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762726.403872:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048904832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762726.403912:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762726.403930:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762726.403951:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762726.659588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762727.299577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762727.683537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762728.323692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762728.707566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762729.347600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762729.475731:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762729.475777:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762729.475795:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762729.475833:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762729.475844:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762729.475852:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762729.475860:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789762729.476146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762726/real 1789762729] req@ffff9811d863ad80 x1876700048905216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762729.476177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048905216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762729.476190:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762729.476196:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762729.476201:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762729.476210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762726/real 1789762729] req@ffff9811d863bb80 x1876700048904832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762729.476224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048904832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762729.476232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762729.476235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762729.476239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048904832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762729.476246:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762726/real 1789762729] req@ffff9811d8639180 x1876700048904960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762729.476258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048904960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762729.476283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762729.476288:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762729.476291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048904960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762729.476299:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762726/real 1789762729] req@ffff9811d8639c00 x1876700048905088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762729.476314:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048905088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762729.476322:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762729.476326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762729.476329:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762729.476336:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762726/real 1789762729] req@ffff9811d863b100 x1876700048904704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762729.476349:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048904704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762729.476357:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762729.476361:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762729.476365:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048904704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762729.731531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762730.371686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762730.755514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762731.395590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762731.523662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762731.523675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762731.523679:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762731.523704:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2181 00000100:00080000:0.0:1789762731.523718:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762731.523728:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762731.523771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048905344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762731.523805:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762731.523809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762731.523811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762731.523816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2181 00000100:00080000:0.0:1789762731.523820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762731.523825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762731.523838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762731.523853:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048905344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762731.523855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048905472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762731.523873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762731.523878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762731.523880:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762731.523884:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2181 00000100:00080000:0.0:1789762731.523888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762731.523893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762731.523919:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762731.523932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048905600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762731.523943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762731.523950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1789762731.523950:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762731.523955:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762731.523961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2181 00000100:00080000:0.0:1789762731.523966:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762731.523971:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762731.523977:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762731.523987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048905728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762731.523997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762731.524001:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762731.524003:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762731.524008:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2181 00000100:00080000:0.0:1789762731.524012:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762731.524014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762731/real 1789762731] req@ffff9811c2e2b800 x1876700048905344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762731.524019:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762731.524024:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762731.524038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048905856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762731.524051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048905344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762731.524066:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762731.524072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762731.524077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048905344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762731.524109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048905472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762731.524137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762731.524152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762731/real 1789762731] req@ffff9811c2e29f80 x1876700048905472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762731.524165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048905472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762731.524174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762731.524177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762731.524181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048905472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762731.524187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762731.524207:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762731.524220:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762731/real 1789762731] req@ffff9811c2e28a80 x1876700048905600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762731.524232:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048905600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762731.524252:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762731.524256:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762731.524260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762731.524266:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762731.524291:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762731.524303:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762731/real 1789762731] req@ffff9811c2e28380 x1876700048905728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762731.524314:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048905728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762731.524323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762731.524326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762731.524329:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762731.524335:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762731.524355:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762731.524366:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762731/real 1789762731] req@ffff9811c2e2aa00 x1876700048905856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762731.524378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048905856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762731.524386:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762731.524389:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762731.524399:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048905856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762731.779634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762732.419716:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762732.803572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762733.443638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762733.827633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762734.467640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762734.851525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762735.491623:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762735.875519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762736.515676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762736.643596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762736.643611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762736.643639:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762736.643678:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2186 00000100:00080000:0.0:1789762736.643693:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762736.643703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762736.643745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048905984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762736.643778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762736.643782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762736.643785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762736.643789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2186 00000100:00080000:0.0:1789762736.643792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762736.643796:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762736.643806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762736.643818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048906112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762736.643830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762736.643834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762736.643836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762736.643840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2186 00000100:00080000:0.0:1789762736.643843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762736.643849:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048905984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762736.643851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762736.643863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762736.643897:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048906240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762736.643932:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762736.643936:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762736.643939:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762736.643945:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2186 00000100:00080000:0.0:1789762736.643952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762736.643961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762736.643964:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762736.643972:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762736.643992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048906368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762736.644005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762736.644010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762736.644012:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762736.644019:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2186 00000100:00080000:0.0:1789762736.644025:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762736.644033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762736/real 1789762736] req@ffff9811c2e2b100 x1876700048905984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762736.644034:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762736.644047:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762736.644069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048906496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762736.644080:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048905984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762736.644103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762736.644111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762736.644120:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048905984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762736.644132:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048906112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762736.644160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762736.644175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762736/real 1789762736] req@ffff9811c2e2ad80 x1876700048906112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762736.644191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048906112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762736.644202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762736.644206:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762736.644210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048906112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762736.644240:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762736.644260:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762736.644289:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762736/real 1789762736] req@ffff9811c2e29880 x1876700048906240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762736.644305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048906240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762736.644315:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762736.644319:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762736.644324:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762736.644331:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762736.644353:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762736.644364:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762736/real 1789762736] req@ffff9811c2e2bb80 x1876700048906368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762736.644378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048906368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762736.644387:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762736.644390:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762736.644394:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762736.644400:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762736.644417:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762736.644428:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762736/real 1789762736] req@ffff9811c2e29180 x1876700048906496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762736.644441:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048906496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762736.644459:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762736.644463:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762736.644468:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762736.899532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762737.539704:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762737.923514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762738.563646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762738.947525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762739.587614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762739.971581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762740.611706:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762740.995521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762741.635568:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762741.763560:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762741.763570:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762741.763575:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762741.763586:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2191 00000100:00080000:1.0:1789762741.763598:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762741.763617:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762741.763661:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048906624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762741.763688:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762741.763691:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762741.763693:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762741.763696:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2191 00000100:00080000:1.0:1789762741.763699:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762741.763701:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762741.763710:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762741.763720:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048906752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762741.763729:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762741.763732:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762741.763741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048906624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762741.763750:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762741.763755:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2191 00000100:00080000:1.0:1789762741.763758:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762741.763761:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762741.763765:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762741.763775:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048906880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762741.763782:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762741.763785:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762741.763786:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762741.763789:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2191 00000100:00080000:1.0:1789762741.763791:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762741.763793:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762741.763796:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762741.763805:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048907008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762741.763812:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762741.763818:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762741.763819:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762741.763824:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762741.763829:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2191 00000100:00080000:1.0:1789762741.763834:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762741.763839:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762741.763845:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762741.763864:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762741/real 1789762741] req@ffff9811c92c7b80 x1876700048906624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762741.763876:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048907136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762741.763883:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048906624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762741.763901:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762741.763905:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762741.763910:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048906624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762741.763934:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048906752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762741.763950:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762741.763961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762741/real 1789762741] req@ffff9811c92c5880 x1876700048906752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762741.763974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048906752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762741.763983:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762741.763987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762741.763990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048906752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762741.763997:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762741.764028:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762741.764037:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762741/real 1789762741] req@ffff9811c92c6d80 x1876700048906880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762741.764044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048906880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762741.764050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762741.764052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762741.764054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048906880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762741.764058:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762741.764070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762741.764076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762741/real 1789762741] req@ffff9811c92c5180 x1876700048907008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762741.764083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048907008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762741.764088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762741.764090:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762741.764092:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762741.764095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762741.764105:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762741.764110:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762741/real 1789762741] req@ffff9811c92c5500 x1876700048907136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762741.764129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048907136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762741.764134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762741.764136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762741.764138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762742.019570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762742.659660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762743.043547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762743.683689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762744.067552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762744.707662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762745.091516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762745.731601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762746.115666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762746.755677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762746.883634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762746.883645:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762746.883649:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762746.883658:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2196 00000100:00080000:0.0:1789762746.883685:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762746.883691:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762746.883721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048907264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762746.883743:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762746.883746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762746.883748:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762746.883750:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2196 00000100:00080000:0.0:1789762746.883753:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762746.883755:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762746.883762:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762746.883771:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048907392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762746.883780:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762746.883782:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762746.883785:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762746.883786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048907264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762746.883789:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2196 00000100:00080000:0.0:1789762746.883795:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762746.883798:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762746.883801:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762746.883810:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048907520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762746.883817:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762746.883824:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762746.883826:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762746.883829:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2196 00000100:00080000:0.0:1789762746.883832:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762746.883835:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762746.883839:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762746.883849:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048907648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762746.883852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762746.883861:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762746.883865:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762746.883866:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762746.883870:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2196 00000100:00080000:0.0:1789762746.883873:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762746.883875:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762746.883878:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762746.883886:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048907776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762746.883894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762746/real 1789762746] req@ffff9811c2e28e00 x1876700048907264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762746.883910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048907264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762746.883927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762746.883931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762746.883934:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048907264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762746.883952:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048907392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762746.883982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762746.883991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762746/real 1789762746] req@ffff9811c2e29c00 x1876700048907392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762746.884000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048907392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762746.884006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762746.884008:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762746.884010:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048907392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762746.884015:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762746.884026:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762746.884032:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762746/real 1789762746] req@ffff9811c2e28700 x1876700048907520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762746.884040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048907520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762746.884045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762746.884047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762746.884049:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762746.884052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762746.884062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762746.884068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762746/real 1789762746] req@ffff9811c2e2a680 x1876700048907648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762746.884086:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048907648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762746.884092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762746.884094:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762746.884096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762746.884100:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762746.884113:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762746.884119:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762746/real 1789762746] req@ffff9811c2e29180 x1876700048907776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762746.884126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048907776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762746.884131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762746.884133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762746.884135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048907776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762747.139545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762747.779630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762748.163533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762748.803653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762749.187518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762749.827650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762750.211517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762750.851647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762751.235581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762751.875631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762752.003595:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762752.003606:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762752.003611:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762752.003620:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2201 00000100:00080000:0.0:1789762752.003632:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762752.003639:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762752.003673:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048907904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762752.003716:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762752.003719:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762752.003721:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762752.003726:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2201 00000100:00080000:0.0:1789762752.003749:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762752.003757:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762752.003772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762752.003795:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048907904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762752.003801:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048908032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762752.003825:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762752.003831:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762752.003834:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762752.003842:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2201 00000100:00080000:0.0:1789762752.003861:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762752.003868:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762752.003875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762752.003883:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762752.003906:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048908160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762752.003939:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762752.003942:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762752.003944:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762752.003948:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2201 00000100:00080000:0.0:1789762752.003960:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762752.003960:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762752/real 1789762752] req@ffff9811c2e2bb80 x1876700048907904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762752.003969:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762752.003980:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762752.003991:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048907904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762752.003997:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048908288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762752.004005:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762752.004013:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762752.004015:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762752.004022:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789762752.004024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048907904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762752.004026:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762752.004036:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2201 00000100:00080000:0.0:1789762752.004041:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762752.004045:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762752.004068:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762752.004079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048908032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762752.004091:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048908416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762752.004123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762752.004144:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762752/real 1789762752] req@ffff9811c2e29880 x1876700048908032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762752.004172:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048908032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762752.004188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762752.004192:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762752.004196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048908032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762752.004204:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762752.004258:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762752.004270:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762752/real 1789762752] req@ffff9811c2e2ad80 x1876700048908160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762752.004292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048908160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762752.004302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762752.004306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762752.004310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762752.004315:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762752.004332:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762752.004360:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762752/real 1789762752] req@ffff9811c2e2b100 x1876700048908288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762752.004372:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048908288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762752.004380:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762752.004383:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762752.004386:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762752.004403:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762752.004439:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762752.004450:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762752/real 1789762752] req@ffff9811c2e2aa00 x1876700048908416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762752.004554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048908416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762752.004566:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762752.004595:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762752.004601:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762752.259540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762752.899694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762753.283559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762753.923636:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762754.307572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762754.947705:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762755.331574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762755.971607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762756.355554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762756.995638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762757.123594:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762757.123602:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762757.123605:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762757.123614:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2206 00000100:00080000:1.0:1789762757.123624:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762757.123630:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762757.123662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048908544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762757.123686:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762757.123689:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762757.123690:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762757.123693:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2206 00000100:00080000:1.0:1789762757.123695:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762757.123698:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762757.123705:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762757.123715:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048908672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762757.123745:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762757.123751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048908544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762757.123752:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762757.123756:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762757.123775:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2206 00000100:00080000:1.0:1789762757.123780:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762757.123784:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762757.123789:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762757.123800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048908800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762757.123808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762757.123811:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762757.123814:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762757.123816:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2206 00000100:00080000:1.0:1789762757.123820:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762757.123822:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762757.123827:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762757.123835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048908928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762757.123843:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762757.123851:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762757.123855:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762757.123857:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762757.123861:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2206 00000100:00080000:1.0:1789762757.123871:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762757.123875:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762757.123879:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762757.123890:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048909056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762757.123926:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762757/real 1789762757] req@ffff9811c92c4000 x1876700048908544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762757.123945:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048908544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762757.123957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762757.123962:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762757.123975:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048908544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762757.124010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048908672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762757.124037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762757.124051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762757/real 1789762757] req@ffff9811c92c4e00 x1876700048908672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762757.124063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048908672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762757.124070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762757.124074:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762757.124078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048908672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762757.124098:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762757.124122:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762757.124135:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762757/real 1789762757] req@ffff9811c92c7480 x1876700048908800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762757.124148:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048908800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762757.124156:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762757.124160:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762757.124163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762757.124169:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762757.124187:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762757.124195:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762757/real 1789762757] req@ffff9811c92c5c00 x1876700048908928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762757.124202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048908928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762757.124207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762757.124209:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762757.124211:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048908928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762757.124215:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762757.124225:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762757.124231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762757/real 1789762757] req@ffff9811c92c6680 x1876700048909056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762757.124245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048909056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762757.124250:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762757.124252:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762757.124254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762757.379571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762758.019605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762758.403570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762759.043552:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762759.427573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762760.067572:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762760.451555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762761.091599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762761.475525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762762.115701:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762762.243620:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762762.243632:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762762.243636:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762762.243648:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2211 00000100:00080000:1.0:1789762762.243661:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762762.243669:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762762.243714:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048909184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762762.243752:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762762.243756:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762762.243759:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762762.243763:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2211 00000100:00080000:1.0:1789762762.243767:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762762.243770:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762762.243780:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762762.243795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048909312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762762.243810:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762762.243816:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048909184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762762.243839:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762762.243843:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762762.243850:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2211 00000100:00080000:1.0:1789762762.243855:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762762.243859:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762762.243866:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762762.243883:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048909440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762762.243896:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762762.243901:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762762.243903:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762762.243906:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2211 00000100:00080000:1.0:1789762762.243910:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762762.243915:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789762762.243924:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789762762.243925:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762762.243943:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048909568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762762.243955:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762762.243959:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762762.243962:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762762.243966:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2211 00000100:00080000:1.0:1789762762.243969:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762762.243973:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762762.243986:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762762.244001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762762/real 1789762762] req@ffff9811c92c4380 x1876700048909184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762762.244005:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048909696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762762.244075:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048909184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762762.244097:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762762.244107:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762762.244116:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048909184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762762.244172:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048909312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762762.244224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762762.244254:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762762/real 1789762762] req@ffff9811c92c4a80 x1876700048909312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762762.244280:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048909312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762762.244292:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762762.244298:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762762.244303:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048909312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762762.244313:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762762.244344:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762762.244359:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762762/real 1789762762] req@ffff9811c92c5f80 x1876700048909440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762762.244373:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048909440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762762.244383:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762762.244388:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762762.244394:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762762.244401:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762762.244424:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762762.244437:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762762/real 1789762762] req@ffff9811c92c6a00 x1876700048909568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762762.245139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048909568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762762.245158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762762.245165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762762.245171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762762.245279:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762762.245321:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762762.245339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762762/real 1789762762] req@ffff9811c92c6300 x1876700048909696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762762.245355:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048909696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762762.245366:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762762.245370:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762762.245374:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048909696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762762.499601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762763.139556:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762763.523551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762764.163667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762764.547571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762765.187585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762765.571593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762766.211765:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762766.595523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762767.235690:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762767.363624:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762767.363636:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762767.363641:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762767.363653:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2216 00000100:00080000:0.0:1789762767.363666:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762767.363675:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762767.363721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048909824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762767.363774:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762767.363779:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762767.363781:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762767.363786:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2216 00000100:00080000:0.0:1789762767.363791:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762767.363795:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762767.363806:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762767.363822:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048909952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762767.363836:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762767.363841:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762767.363843:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762767.363850:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048909824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762767.363852:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2216 00000100:00080000:0.0:1789762767.363861:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762767.363866:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762767.363871:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762767.363894:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048910080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762767.363914:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762767.363920:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762767.363923:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762767.363930:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2216 00000800:00000100:3.0:1789762767.363951:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762767.363952:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762767.363966:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762767.363978:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762767.364003:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048910208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762767.364014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762767/real 1789762767] req@ffff9811c2e28380 x1876700048909824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762767.364021:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762767.364033:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762767.364038:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762767.364050:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2216 00000100:00080000:0.0:1789762767.364058:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762767.364063:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762767.364070:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762767.364076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048909824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762767.364088:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048910336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762767.364108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762767.364117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762767.364124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048909824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762767.364153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048909952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762767.364182:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762767.364197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762767/real 1789762767] req@ffff9811c2e28a80 x1876700048909952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762767.364211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048909952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762767.364220:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762767.364225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762767.364228:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048909952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762767.364234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762767.364250:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762767.364260:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762767/real 1789762767] req@ffff9811c2e29f80 x1876700048910080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762767.364289:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048910080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762767.364298:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762767.364302:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762767.364305:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762767.364311:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762767.364346:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762767.364358:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762767/real 1789762767] req@ffff9811c2e2b800 x1876700048910208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762767.364371:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048910208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762767.364379:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762767.364382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762767.364385:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762767.364391:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762767.364409:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762767.364447:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762767/real 1789762767] req@ffff9811c2e2b480 x1876700048910336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762767.364492:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048910336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762767.364527:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762767.364531:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762767.364535:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762767.619601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762768.259615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762768.643563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762769.283772:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762769.667540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762770.307601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762770.691594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762771.332556:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762771.715526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762772.355662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762772.483673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762772.483685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762772.483690:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762772.483703:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2222 00000100:00080000:2.0:1789762772.483715:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762772.483724:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762772.483786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048910464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762772.483826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762772.483830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762772.483832:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762772.483836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2222 00000100:00080000:2.0:1789762772.483841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762772.483845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762772.483858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762772.483872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048910592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762772.483892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762772.483896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762772.483897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048910464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762772.483898:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762772.483902:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2222 00000100:00080000:2.0:1789762772.483928:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762772.483932:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762772.483938:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762772.483948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048910720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762772.483958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762772.483962:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762772.483964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762772.483967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2222 00000100:00080000:2.0:1789762772.483970:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762772.483974:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762772.483980:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762772.483982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762772.483988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048910848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762772.483996:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762772.484000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762772.484001:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762772.484004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2222 00000100:00080000:2.0:1789762772.484007:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762772.484011:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762772.484014:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762772.484022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048910976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762772.484044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762772/real 1789762772] req@ffff9811d8639880 x1876700048910464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762772.484065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048910464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762772.484077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762772.484081:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762772.484085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048910464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762772.484128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048910592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762772.484154:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762772.484180:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762772/real 1789762772] req@ffff9811d8639500 x1876700048910592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762772.484197:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048910592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762772.484203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762772.484205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762772.484207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048910592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762772.484211:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762772.484228:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762772.484236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762772/real 1789762772] req@ffff9811d8639f80 x1876700048910720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762772.484243:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048910720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762772.484248:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762772.484250:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762772.484251:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762772.484255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762772.484267:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762772.484279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762772/real 1789762772] req@ffff9811d863a680 x1876700048910848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762772.484286:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048910848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762772.484301:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762772.484312:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762772.484315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762772.484319:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762772.484335:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762772.484342:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762772/real 1789762772] req@ffff9811d8638a80 x1876700048910976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762772.484350:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048910976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762772.484355:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762772.484357:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762772.484359:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048910976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762772.739597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762773.379628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762773.763515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762774.403611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762774.787573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762775.427631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762775.811549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762776.451635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762776.835549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762777.475660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762777.603567:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762777.603577:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762777.603582:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762777.603594:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2227 00000100:00080000:1.0:1789762777.603607:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762777.603615:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762777.603656:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048911104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762777.603682:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762777.603685:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762777.603686:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762777.603690:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2227 00000100:00080000:1.0:1789762777.603693:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762777.603696:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762777.603706:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762777.603714:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048911232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762777.603724:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762777.603727:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762777.603746:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762777.603747:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048911104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762777.603758:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2227 00000100:00080000:1.0:1789762777.603764:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762777.603768:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762777.603773:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762777.603792:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048911360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762777.603801:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762777.603804:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762777.603806:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762777.603809:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2227 00000100:00080000:1.0:1789762777.603811:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762777.603814:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762777.603817:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762777.603828:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048911488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762777.603836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762777.603848:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762777.603851:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762777.603854:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762777.603859:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2227 00000100:00080000:1.0:1789762777.603862:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762777.603865:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762777.603869:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762777.603887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762777/real 1789762777] req@ffff9811c92c4700 x1876700048911104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762777.603888:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048911616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762777.603984:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048911104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762777.603993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762777.603997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762777.604000:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048911104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762777.604006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048911232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762777.604025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762777.604034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762777/real 1789762777] req@ffff9811c92c7800 x1876700048911232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762777.604042:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048911232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762777.604047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762777.604049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762777.604051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048911232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762777.604072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762777.604083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762777.604090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762777/real 1789762777] req@ffff9811c92c7100 x1876700048911360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762777.604098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048911360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762777.604112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762777.604115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762777.604117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762777.604121:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762777.604135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762777.604142:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762777/real 1789762777] req@ffff9811c92c6300 x1876700048911488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762777.604149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048911488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762777.604154:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762777.604156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762777.604158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762777.604161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762777.604170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762777.604176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762777/real 1789762777] req@ffff9811c92c6a00 x1876700048911616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762777.604183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048911616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762777.604188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762777.604190:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762777.604197:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048911616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762777.859581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762778.499610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762778.883569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762779.523633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762779.907549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762780.547612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762780.931561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762781.571610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762781.955581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762782.595748:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762782.723612:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762782.723623:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762782.723627:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762782.723638:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2232 00000100:00080000:0.0:1789762782.723650:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762782.723681:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762782.723721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048911744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762782.723749:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762782.723752:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762782.723753:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762782.723756:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2232 00000100:00080000:0.0:1789762782.723759:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762782.723761:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762782.723770:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762782.723779:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048911872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762782.723788:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762782.723791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762782.723793:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762782.723795:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2232 00000100:00080000:0.0:1789762782.723796:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762782.723799:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762782.723803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762782.723811:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048912000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762782.723818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762782.723825:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1789762782.723827:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048911744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762782.723836:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762782.723845:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2232 00000100:00080000:0.0:1789762782.723848:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762782.723851:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762782.723857:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762782.723868:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048912128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762782.723875:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762782.723877:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762782.723878:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762782.723881:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2232 00000100:00080000:0.0:1789762782.723884:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762782.723887:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762782.723890:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762782.723897:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048912256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762782.723906:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762782.723954:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762782/real 1789762782] req@ffff9811c2e2a300 x1876700048911744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762782.723969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048911744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762782.723978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762782.723981:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762782.723984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048911744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762782.723990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048911872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762782.724006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762782.724014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762782/real 1789762782] req@ffff9811c2e28000 x1876700048911872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762782.724022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048911872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762782.724028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762782.724030:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762782.724046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048911872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762782.724067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762782.724083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762782.724090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762782/real 1789762782] req@ffff9811c2e29500 x1876700048912000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762782.724097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048912000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762782.724103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762782.724105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762782.724106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762782.724110:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762782.724122:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762782.724128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762782/real 1789762782] req@ffff9811c2e2b480 x1876700048912128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762782.724135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048912128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762782.724140:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762782.724141:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762782.724143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762782.724147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762782.724156:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762782.724168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762782/real 1789762782] req@ffff9811c2e2b800 x1876700048912256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762782.724177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048912256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762782.724182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762782.724184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762782.724185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762782.979612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762783.619600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762784.003548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762784.643714:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762785.027541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762785.667595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762786.051575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762786.691639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762787.075560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762787.715607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762787.843586:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762787.843598:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762787.843603:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762787.843614:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2237 00000100:00080000:1.0:1789762787.843631:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762787.843638:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762787.843676:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048912384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762787.843700:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762787.843703:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762787.843705:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762787.843708:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2237 00000100:00080000:1.0:1789762787.843711:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762787.843713:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762787.843721:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762787.843731:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048912512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762787.843742:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762787.843745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048912384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762787.843759:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762787.843761:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762787.843766:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2237 00000100:00080000:1.0:1789762787.843770:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762787.843782:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762787.843787:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762787.843798:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048912640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762787.843806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762787.843815:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762787.843819:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762787.843820:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762787.843825:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2237 00000100:00080000:1.0:1789762787.843828:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762787.843831:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762787.843835:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762787.843845:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048912768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762787.843849:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762787/real 1789762787] req@ffff9811c92c5f80 x1876700048912384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762787.843862:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762787.843869:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762787.843871:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762787.843877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048912384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762787.843878:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2237 00000100:00080000:1.0:1789762787.843884:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762787.843892:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789762787.843897:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762787.843906:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789762787.843907:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762787.843911:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048912384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762787.843922:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048912896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762787.843938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048912512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762787.843969:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762787.843996:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762787/real 1789762787] req@ffff9811c92c4a80 x1876700048912512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762787.844012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048912512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762787.844020:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762787.844023:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762787.844026:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048912512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762787.844031:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762787.844048:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762787.844057:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762787/real 1789762787] req@ffff9811c92c4380 x1876700048912640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762787.844065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048912640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762787.844071:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762787.844073:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762787.844075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762787.844079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762787.844090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762787.844096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762787/real 1789762787] req@ffff9811c92c6680 x1876700048912768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762787.844104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048912768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762787.844119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762787.844123:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762787.844127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762787.844134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762787.844161:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762787.844174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762787/real 1789762787] req@ffff9811c92c5c00 x1876700048912896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762787.844188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048912896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762787.844197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762787.844200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762787.844202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048912896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762788.099537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762788.739613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762789.123550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762789.763639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762790.147577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762790.787635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762791.171571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762791.811646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762792.195548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789762792.835603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762792.963568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762792.963577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762792.963581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762792.963589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2242 00000100:00080000:3.0:1789762792.963599:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762792.963605:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762792.963651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048913024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762792.963681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762792.963684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762792.963686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762792.963688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2242 00000100:00080000:3.0:1789762792.963691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762792.963694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762792.963701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762792.963709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048913152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762792.963717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762792.963720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762792.963721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762792.963723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2242 00000100:00080000:3.0:1789762792.963725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762792.963727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762792.963729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762792.963735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048913280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762792.963743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762792.963746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762792.963748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762792.963757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2242 00000100:00100000:1.0:1789762792.963759:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048913024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762792.963761:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762792.963768:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762792.963773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762792.963784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048913408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762792.963791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762792.963794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762792.963796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762792.963798:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2242 00000100:00080000:3.0:1789762792.963801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762792.963803:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762792.963806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762792.963813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048913536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762792.963855:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048913152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762792.963876:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762792.963897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762792.963915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762793.220697:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762793.859661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762794.243700:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762794.883633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762795.267587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762795.907630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789762796.035869:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789762796.035946:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762796.035972:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762796.036030:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762796.036046:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762796.036059:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789762796.036089:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789762796.036233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762792/real 1789762796] req@ffff9811e1a23800 x1876700048913408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762796.036279:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048913408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762796.036300:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762796.036308:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762796.036315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762796.036328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762792/real 1789762796] req@ffff9811e1a23100 x1876700048913536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762796.036415:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048913536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762796.036432:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762796.036438:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762796.036445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762796.036457:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762792/real 1789762796] req@ffff9811e1a20700 x1876700048913024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762796.036521:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048913024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762796.036535:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762796.036542:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762796.036548:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048913024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762796.036561:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762792/real 1789762796] req@ffff9811e1a21500 x1876700048913152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762796.036587:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048913152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762796.036602:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762796.036608:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762796.036614:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048913152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762796.036626:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762792/real 1789762796] req@ffff9811e1a23480 x1876700048913280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762796.036664:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048913280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762796.036679:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762796.036685:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762796.036692:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762796.291567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762796.931611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762797.315523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762797.955626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762798.083611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762798.083621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762798.083625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762798.083654:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2247 00000100:00080000:3.0:1789762798.083665:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762798.083675:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762798.083716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048913664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762798.083742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762798.083745:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762798.083747:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762798.083752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2247 00000100:00080000:3.0:1789762798.083759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762798.083764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762798.083773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762798.083798:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048913664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762798.083806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048913792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762798.083828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762798.083833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762798.083836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762798.083842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2247 00000100:00080000:3.0:1789762798.083848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762798.083855:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762798.083861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762798.083877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048913920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762798.083907:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762798.083923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762798.083929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762798.083931:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762798.083938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2247 00000100:00080000:3.0:1789762798.083944:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762798.083950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762798.083956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762798.083974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048914048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762798.083985:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762798.083991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762798/real 1789762798] req@ffff9811e1a22300 x1876700048913664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762798.083994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762798.083999:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762798.084015:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2247 00000100:00080000:3.0:1789762798.084024:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762798.084035:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762798.084047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048913664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762798.084049:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762798.084074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048914176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762798.084075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762798.084091:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762798.084100:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048913664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762798.084148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048913792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762798.084179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762798.084215:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762798/real 1789762798] req@ffff9811e1a21180 x1876700048913792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762798.084228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048913792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762798.084256:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762798.084260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762798.084264:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048913792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762798.084271:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762798.084296:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762798.084308:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762798/real 1789762798] req@ffff9811e1a22680 x1876700048913920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762798.084321:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048913920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762798.084330:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762798.084334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762798.084337:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048913920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762798.084343:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762798.084365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762798.084377:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762798/real 1789762798] req@ffff9811e1a21c00 x1876700048914048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762798.084391:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048914048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762798.084400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762798.084403:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762798.084415:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762798.084424:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762798.084449:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762798.084526:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762798/real 1789762798] req@ffff9811e1a21880 x1876700048914176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762798.084543:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048914176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762798.084552:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762798.084556:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762798.084560:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762798.339527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762798.979569:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762799.363521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762800.003677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762800.387511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762801.027700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762801.411531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762802.051666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762802.435526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762803.075618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762803.203548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762803.203559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762803.203563:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762803.203574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2252 00000100:00080000:0.0:1789762803.203608:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762803.203616:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762803.203664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048914304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762803.203688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762803.203690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762803.203692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762803.203695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2252 00000100:00080000:0.0:1789762803.203698:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762803.203701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762803.203712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762803.203722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048914432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762803.203727:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048914304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762803.203736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762803.203739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762803.203740:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762803.203744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2252 00000100:00080000:0.0:1789762803.203746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762803.203748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762803.203752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762803.203759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048914560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762803.203767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762803.203774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762803.203775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762803.203779:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2252 00000100:00080000:0.0:1789762803.203803:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762803.203806:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762803.203810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762803.203814:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762803.203820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048914688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762803.203828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762803.203830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762803.203832:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762803.203835:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2252 00000100:00080000:0.0:1789762803.203838:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762803.203841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762803.203844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762803.203851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048914816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762803.203865:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762803/real 1789762803] req@ffff9811c2e29f80 x1876700048914304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762803.203888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048914304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762803.203897:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762803.203900:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762803.203903:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048914304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762803.203927:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048914432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762803.203952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762803.203961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762803/real 1789762803] req@ffff9811c2e28a80 x1876700048914432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762803.203969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048914432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762803.203975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762803.203977:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762803.203979:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048914432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762803.203983:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762803.203994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762803.203999:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762803/real 1789762803] req@ffff9811c2e28380 x1876700048914560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762803.204007:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048914560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762803.204012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762803.204013:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762803.204015:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762803.204018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762803.204028:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762803.204034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762803/real 1789762803] req@ffff9811c2e2aa00 x1876700048914688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762803.204050:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048914688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762803.204056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762803.204058:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762803.204060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762803.204063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762803.204075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762803.204081:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762803/real 1789762803] req@ffff9811c2e2b100 x1876700048914816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762803.204088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048914816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762803.204093:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762803.204094:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762803.204096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048914816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762803.459505:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762804.099627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762804.483512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762805.123612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762805.507565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762806.147581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762806.531536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762807.171619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762807.555563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762808.195558:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762808.323540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762808.323550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762808.323555:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762808.323565:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2257 00000100:00080000:0.0:1789762808.323579:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762808.323588:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762808.323639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048914944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762808.323684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762808.323687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762808.323689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762808.323692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2257 00000100:00080000:0.0:1789762808.323695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762808.323698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762808.323707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762808.323718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048915072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762808.323727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762808.323730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762808.323734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762808.323735:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048914944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762808.323745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2257 00000100:00080000:0.0:1789762808.323749:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762808.323753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762808.323759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762808.323773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048915200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762808.323784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762808.323788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762808.323790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762808.323795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2257 00000100:00080000:0.0:1789762808.323807:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762808.323809:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762808.323814:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762808.323820:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762808.323831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048915328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762808.323839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762808.323842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762808.323844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762808.323849:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2257 00000100:00080000:0.0:1789762808.323853:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762808.323859:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762808/real 1789762808] req@ffff9811c2e2ad80 x1876700048914944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762808.323860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762808.323870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762808.323881:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048914944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762808.323888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048915456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762808.323908:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762808.323912:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762808.323916:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048914944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762808.323939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048915072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762808.323957:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762808.323966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762808/real 1789762808] req@ffff9811c2e29880 x1876700048915072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762808.323974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048915072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762808.323980:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762808.323982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762808.323984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048915072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762808.323987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762808.323997:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762808.324003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762808/real 1789762808] req@ffff9811c2e2bb80 x1876700048915200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762808.324010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048915200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762808.324015:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762808.324017:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762808.324019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762808.324027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762808.324041:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762808.324047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762808/real 1789762808] req@ffff9811c2e29180 x1876700048915328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762808.324054:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048915328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762808.324060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762808.324062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762808.324065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762808.324068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762808.324078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762808.324083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762808/real 1789762808] req@ffff9811c2e2a680 x1876700048915456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762808.324090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048915456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762808.324102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762808.324104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762808.324117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762808.579546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762809.219602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762809.603565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762810.243614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762810.627527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762811.267603:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762811.651550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762812.291563:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762812.675552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762813.315662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762813.443560:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762813.443571:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762813.443575:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762813.443586:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2263 00000100:00080000:3.0:1789762813.443598:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762813.443605:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762813.443647:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048915584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762813.443672:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762813.443675:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762813.443676:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762813.443698:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2263 00000100:00080000:3.0:1789762813.443701:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762813.443704:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762813.443714:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762813.443726:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048915712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762813.443737:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762813.443740:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762813.443742:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762813.443749:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2263 00000100:00100000:1.0:1789762813.443750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048915584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762813.443752:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762813.443761:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762813.443767:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762813.443778:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048915840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762813.443785:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762813.443788:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762813.443789:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762813.443793:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2263 00000100:00080000:3.0:1789762813.443796:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762813.443800:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762813.443804:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762813.443813:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048915968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762813.443824:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762813.443829:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1789762813.443832:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762813.443833:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762813.443839:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2263 00000100:00080000:3.0:1789762813.443845:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762813.443850:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762813.443855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762813.443866:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048916096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762813.443890:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762813/real 1789762813] req@ffff9811e1a22a00 x1876700048915584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762813.443927:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048915584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762813.443939:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762813.443944:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762813.443948:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048915584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762813.443975:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048915712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762813.444003:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762813.444017:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762813/real 1789762813] req@ffff9811e1a22d80 x1876700048915712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762813.444030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048915712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762813.444039:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762813.444042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762813.444045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048915712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762813.444052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762813.444087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762813.444100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762813/real 1789762813] req@ffff9811e1a21880 x1876700048915840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762813.444114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048915840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762813.444137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762813.444141:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762813.444145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762813.444153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762813.444178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762813.444192:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762813/real 1789762813] req@ffff9811e1a21c00 x1876700048915968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762813.444204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048915968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762813.444213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762813.444216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762813.444220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048915968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762813.444227:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762813.444249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762813.444261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762813/real 1789762813] req@ffff9811e1a22680 x1876700048916096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762813.444274:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048916096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762813.444283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762813.444286:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762813.444297:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762813.699591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762814.339572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762814.723534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762815.363566:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762815.747578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762816.387583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762816.771546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762817.411628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762817.795550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762818.435609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762818.563526:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762818.563541:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762818.563544:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762818.563553:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2268 00000100:00080000:0.0:1789762818.563563:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762818.563569:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762818.563603:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048916224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762818.563634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762818.563663:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762818.563665:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762818.563673:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2268 00000100:00080000:0.0:1789762818.563680:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762818.563686:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762818.563696:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762818.563710:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048916224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762818.563712:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048916352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762818.563737:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762818.563740:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762818.563742:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762818.563747:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2268 00000100:00080000:0.0:1789762818.563751:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762818.563754:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762818.563757:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762818.563766:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048916480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762818.563774:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762818.563778:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762818.563781:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762818.563783:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2268 00000100:00080000:0.0:1789762818.563799:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762818.563803:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762818.563808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762818.563819:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762818.563833:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048916608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762818.563841:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762818.563844:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762818.563846:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762818.563848:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2268 00000100:00080000:0.0:1789762818.563853:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762818.563857:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762818.563862:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762818.563885:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762818/real 1789762818] req@ffff9811c2e28700 x1876700048916224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762818.563886:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048916736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762818.563956:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048916224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762818.563969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762818.563974:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762818.563977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048916224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762818.564003:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048916352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762818.564024:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762818.564051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762818/real 1789762818] req@ffff9811c2e29c00 x1876700048916352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762818.564061:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048916352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762818.564067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762818.564069:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762818.564071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048916352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762818.564076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762818.564092:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762818.564124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762818/real 1789762818] req@ffff9811c2e28e00 x1876700048916480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762818.564137:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048916480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762818.564147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762818.564150:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762818.564154:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762818.564159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762818.564177:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762818.564185:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762818/real 1789762818] req@ffff9811c2e2a680 x1876700048916608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762818.564193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048916608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762818.564198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762818.564200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762818.564202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762818.564206:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762818.564216:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762818.564222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762818/real 1789762818] req@ffff9811c2e29180 x1876700048916736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762818.564229:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048916736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762818.564261:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762818.564263:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762818.564266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048916736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762818.819536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762819.459610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762819.843525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762820.483692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762820.867552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762821.507593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762821.891547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762822.531623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762822.915530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762823.555582:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762823.683583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762823.683616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762823.683622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762823.683633:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2273 00000100:00080000:3.0:1789762823.683644:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762823.683653:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762823.683694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048916864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762823.683719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762823.683722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762823.683723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762823.683726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2273 00000100:00080000:3.0:1789762823.683729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762823.683731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762823.683741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762823.683750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048916992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762823.683758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762823.683763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789762823.683764:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048916864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762823.683765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762823.683772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2273 00000100:00080000:3.0:1789762823.683777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762823.683779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762823.683782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762823.683789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048917120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762823.683801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762823.683803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762823.683805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762823.683808:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2273 00000100:00080000:3.0:1789762823.683811:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762823.683814:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762823.683818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762823.683825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048917248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762823.683831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762823.683834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762823.683835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762823.683837:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2273 00000100:00080000:3.0:1789762823.683839:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762823.683841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762823.683844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762823.683850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762823.683852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048917376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762823.683904:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762823/real 1789762823] req@ffff9811e1a21180 x1876700048916864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762823.683924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048916864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762823.683935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762823.683939:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762823.683944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048916864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762823.683971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048916992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762823.684007:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762823.684021:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762823/real 1789762823] req@ffff9811e1a22300 x1876700048916992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762823.684032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048916992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762823.684040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762823.684044:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762823.684047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048916992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762823.684060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762823.684080:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762823.684090:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762823/real 1789762823] req@ffff9811e1a23480 x1876700048917120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762823.684100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048917120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762823.684107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762823.684110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762823.684113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762823.684119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762823.684137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762823.684146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762823/real 1789762823] req@ffff9811e1a21500 x1876700048917248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762823.684157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048917248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762823.684164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762823.684167:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762823.684171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762823.684176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762823.684193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762823.684203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762823/real 1789762823] req@ffff9811e1a20700 x1876700048917376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762823.684222:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048917376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762823.684229:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762823.684232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762823.684235:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762823.939573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762824.579601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762824.963525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762825.603611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762825.987535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762826.627590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762827.011607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762827.651611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762828.035516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762828.675644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762828.803616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762828.803631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762828.803636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762828.803649:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2278 00000100:00080000:0.0:1789762828.803663:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762828.803672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762828.803721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048917504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762828.803753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762828.803758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762828.803759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762828.803764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2278 00000100:00080000:0.0:1789762828.803786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762828.803793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762828.803805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762828.803822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048917632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762828.803835:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048917504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762828.803845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762828.803851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762828.803853:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762828.803861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2278 00000100:00080000:0.0:1789762828.803866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762828.803872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762828.803878:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762828.803892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048917760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762828.803903:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762828.803904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762828.803910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762828.803912:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762828.803919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2278 00000100:00080000:0.0:1789762828.803925:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762828.803932:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762828.803939:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762828.803947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762828/real 1789762828] req@ffff9811c2e2bb80 x1876700048917504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762828.803972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048917888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762828.803982:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048917504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762828.803998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762828.804007:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762828.804019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048917504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762828.804030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762828.804035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762828.804037:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762828.804045:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2278 00000100:00100000:1.0:1789762828.804051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048917632:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789762828.804055:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762828.804064:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762828.804072:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789762828.804078:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762828.804103:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048918016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762828.804104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762828/real 1789762828] req@ffff9811c2e29880 x1876700048917632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762828.804178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048917632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762828.804206:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762828.804210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762828.804213:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048917632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762828.804220:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762828.804237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762828.804252:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762828/real 1789762828] req@ffff9811c2e2ad80 x1876700048917760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762828.804263:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048917760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762828.804269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762828.804271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762828.804273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762828.804277:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762828.804289:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762828.804296:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762828/real 1789762828] req@ffff9811c2e2b100 x1876700048917888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762828.804303:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048917888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762828.804308:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762828.804310:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762828.804318:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048917888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762828.804342:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762828.804367:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762828.804379:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762828/real 1789762828] req@ffff9811c2e2aa00 x1876700048918016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762828.804393:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048918016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762828.804402:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762828.804405:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762828.804409:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762829.059588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762829.699641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762830.083525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762830.723629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762831.107603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762831.747616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762832.131527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762832.771627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762833.155567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762833.795589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762833.923578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762833.923590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762833.923596:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762833.923609:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2283 00000100:00080000:0.0:1789762833.923624:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762833.923633:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762833.923683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048918144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762833.923710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762833.923714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762833.923716:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762833.923721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2283 00000100:00080000:0.0:1789762833.923725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762833.923730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762833.923745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048918144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762833.923763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762833.923781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048918272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762833.923795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762833.923799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762833.923801:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762833.923805:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2283 00000100:00080000:0.0:1789762833.923818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762833.923823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762833.923829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762833.923832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762833.923846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048918400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762833.923856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762833.923860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762833.923862:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762833.923867:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2283 00000100:00080000:0.0:1789762833.923871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762833.923876:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762833.923882:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762833/real 1789762833] req@ffff9811c2e28380 x1876700048918144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762833.923884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762833.923900:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048918528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762833.923910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762833.923913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789762833.923928:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048918144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762833.923936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762833.923947:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2283 00000100:00080000:1.0:1789762833.923948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762833.923956:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762833.923956:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762833.923962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048918144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762833.923966:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762833.923974:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762833.923987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048918656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762833.923992:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048918272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762833.924027:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762833.924048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762833/real 1789762833] req@ffff9811c2e28a80 x1876700048918272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762833.924058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048918272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762833.924064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762833.924067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762833.924070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048918272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762833.924075:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762833.924095:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762833.924104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762833/real 1789762833] req@ffff9811c2e29f80 x1876700048918400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762833.924111:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048918400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762833.924117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762833.924119:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762833.924121:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762833.924125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762833.924139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762833.924146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762833/real 1789762833] req@ffff9811c2e2b800 x1876700048918528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762833.924154:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048918528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762833.924168:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762833.924170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762833.924174:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762833.924189:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762833.924208:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762833.924216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762833/real 1789762833] req@ffff9811c2e2b480 x1876700048918656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762833.924226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048918656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762833.924232:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762833.924235:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762833.924240:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048918656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762834.179531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762834.819614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762835.203570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762835.843584:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762836.227566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762836.867611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762837.251551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762837.891568:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762838.275528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762838.915633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762839.043589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762839.043601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762839.043606:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762839.043617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2288 00000100:00080000:1.0:1789762839.043629:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762839.043637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762839.043676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048918784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762839.043729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762839.043734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762839.043736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762839.043741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2288 00000100:00080000:1.0:1789762839.043746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762839.043753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762839.043764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762839.043783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048918912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762839.043786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048918784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762839.043812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762839.043817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762839.043819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762839.043824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2288 00000100:00080000:1.0:1789762839.043829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762839.043834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762839.043840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762839.043857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048919040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762839.043868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762839.043873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762839.043876:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762839.043892:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762839.043900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2288 00000100:00080000:1.0:1789762839.043914:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762839.043921:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762839.043927:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762839.043941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048919168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762839.043954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762839.043958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762839.043961:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762839.043965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762839/real 1789762839] req@ffff9811c92c7480 x1876700048918784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762839.043970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2288 00000100:00080000:1.0:1789762839.043983:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762839.043995:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762839.044004:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762839.044004:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048918784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762839.044022:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048919296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762839.044041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762839.044050:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762839.044055:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048918784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762839.044082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048918912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762839.044103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762839.044113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762839/real 1789762839] req@ffff9811c92c4e00 x1876700048918912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762839.044121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048918912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762839.044127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762839.044129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762839.044131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048918912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762839.044135:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762839.044146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762839.044152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762839/real 1789762839] req@ffff9811c92c4000 x1876700048919040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762839.044159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048919040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762839.044164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762839.044166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762839.044168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762839.044177:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762839.044192:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762839.044198:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762839/real 1789762839] req@ffff9811c92c5500 x1876700048919168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762839.044206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048919168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762839.044211:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762839.044213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762839.044214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762839.044218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762839.044228:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762839.044233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762839/real 1789762839] req@ffff9811c92c5180 x1876700048919296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762839.044240:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048919296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762839.044245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762839.044247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762839.044249:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762839.299551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762839.939609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762840.323597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762840.963684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762841.347516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762841.987608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762842.371544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762843.011621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762843.395540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762844.035580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762844.163610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762844.163620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762844.163625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762844.163636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2293 00000100:00080000:0.0:1789762844.163649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762844.163657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762844.163702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048919424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762844.163727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762844.163730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762844.163732:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762844.163735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2293 00000100:00080000:0.0:1789762844.163738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762844.163741:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762844.163750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762844.163759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048919552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762844.163770:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048919424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762844.163793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762844.163798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762844.163800:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762844.163806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2293 00000100:00080000:0.0:1789762844.163811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762844.163816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762844.163822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762844.163836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048919680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762844.163857:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762844.163863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762844.163868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762844.163871:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762844.163876:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2293 00000100:00080000:0.0:1789762844.163886:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762844.163891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762844.163897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762844.163917:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762844/real 1789762844] req@ffff9811c2e29500 x1876700048919424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762844.163918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048919808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762844.163930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762844.163933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762844.163936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762844.163943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2293 00000100:00080000:0.0:1789762844.163955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762844.163959:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048919424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762844.163963:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762844.163973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762844.163976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762844.163988:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762844.163993:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048919424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762844.163996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048919936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762844.164023:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048919552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762844.164044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762844.164053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762844/real 1789762844] req@ffff9811c2e28000 x1876700048919552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762844.164062:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048919552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762844.164074:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762844.164076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762844.164078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048919552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762844.164082:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762844.164095:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762844.164102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762844/real 1789762844] req@ffff9811c2e2a300 x1876700048919680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762844.164109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048919680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762844.164114:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762844.164115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762844.164117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762844.164121:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762844.164140:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762844.164146:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762844/real 1789762844] req@ffff9811c2e2b480 x1876700048919808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762844.164154:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048919808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762844.164159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762844.164160:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762844.164162:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762844.164166:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762844.164176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762844.164181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762844/real 1789762844] req@ffff9811c2e2b800 x1876700048919936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762844.164188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048919936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762844.164193:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762844.164194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762844.164196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048919936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762844.419543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762845.059567:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762845.443555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762846.083554:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762846.467521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762847.107657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762847.491577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762848.131617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762848.515580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762849.155604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762849.283616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762849.283635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762849.283641:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762849.283656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2298 00000100:00080000:0.0:1789762849.283671:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762849.283681:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762849.283731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048920064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762849.283756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762849.283759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762849.283761:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762849.283764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2298 00000100:00080000:0.0:1789762849.283768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762849.283771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762849.283782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762849.283789:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048920064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762849.283796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048920192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762849.283824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762849.283851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762849.283854:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762849.283858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2298 00000100:00080000:0.0:1789762849.283870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762849.283874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789762849.283879:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762849.283882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762849.283896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048920320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762849.283906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762849.283909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762849.283915:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762849.283919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2298 00000100:00080000:0.0:1789762849.283922:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762849.283925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762849.283930:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762849.283939:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762849/real 1789762849] req@ffff9811c2e29f80 x1876700048920064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762849.283941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048920448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762849.283971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762849.283973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762849.283975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762849.283978:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2298 00000100:00080000:0.0:1789762849.283980:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762849.283981:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048920064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762849.283985:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789762849.283994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789762849.283996:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789762849.284001:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762849.284005:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048920064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762849.284013:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048920576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762849.284034:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048920192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762849.284054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762849.284062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762849/real 1789762849] req@ffff9811c2e28a80 x1876700048920192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762849.284070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048920192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762849.284076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762849.284078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762849.284080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048920192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762849.284083:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762849.284094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762849.284100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762849/real 1789762849] req@ffff9811c2e28380 x1876700048920320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762849.284107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048920320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762849.284121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762849.284123:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762849.284125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762849.284129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762849.284143:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762849.284149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762849/real 1789762849] req@ffff9811c2e2aa00 x1876700048920448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762849.284157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048920448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762849.284162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762849.284163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762849.284165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762849.284169:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762849.284178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762849.284184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762849/real 1789762849] req@ffff9811c2e2b100 x1876700048920576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762849.284191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048920576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762849.284195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762849.284197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762849.284205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762849.539575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762850.179625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762850.563566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762851.203641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762851.587523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762852.227561:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762852.611574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762853.251653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762853.635510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762854.275553:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762854.403594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762854.403606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762854.403610:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762854.403625:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2303 00000100:00080000:1.0:1789762854.403638:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762854.403645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762854.403690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048920704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762854.403724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762854.403727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762854.403728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762854.403731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2303 00000100:00080000:1.0:1789762854.403733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762854.403736:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762854.403746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762854.403755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048920832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762854.403780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762854.403788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762854.403791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762854.403791:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048920704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762854.403806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2303 00000100:00080000:1.0:1789762854.403828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762854.403833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762854.403839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762854.403849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048920960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762854.403857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762854.403860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762854.403861:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762854.403869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2303 00000800:00000100:2.0:1789762854.403869:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762854.403879:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762854.403885:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762854.403893:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762854.403907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048921088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762854.403914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762854.403916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762854.403917:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762854.403919:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2303 00000100:00080000:1.0:1789762854.403924:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762854.403927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762854.403932:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762854/real 1789762854] req@ffff9811c92c6d80 x1876700048920704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762854.403933:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762854.403950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048921216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762854.403967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048920704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762854.403983:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762854.403987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762854.403992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048920704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762854.404019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048920832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762854.404036:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762854.404045:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762854/real 1789762854] req@ffff9811c92c5880 x1876700048920832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762854.404053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048920832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762854.404059:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762854.404061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762854.404063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048920832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762854.404067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762854.404104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762854.404111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762854/real 1789762854] req@ffff9811c92c7b80 x1876700048920960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762854.404120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048920960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762854.404126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762854.404128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762854.404130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048920960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762854.404134:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762854.404146:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762854.404151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762854/real 1789762854] req@ffff9811c92c5180 x1876700048921088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762854.404159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048921088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762854.404164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762854.404166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762854.404168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762854.404171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762854.404181:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762854.404186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762854/real 1789762854] req@ffff9811c92c5500 x1876700048921216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762854.404204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048921216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762854.404209:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762854.404212:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762854.404214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762854.659520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762855.299608:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762855.683587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762856.323555:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762856.707566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762857.347582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762857.731563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762858.371595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762858.755579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762859.395590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762859.523596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762859.523605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762859.523609:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762859.523617:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2309 00000100:00080000:0.0:1789762859.523627:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762859.523633:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762859.523666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048921344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762859.523691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762859.523694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762859.523695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762859.523698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2309 00000100:00080000:0.0:1789762859.523700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762859.523703:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762859.523709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762859.523718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048921472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762859.523747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762859.523750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762859.523751:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762859.523757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2309 00000100:00080000:0.0:1789762859.523786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762859.523789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762859.523795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762859.523805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048921600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762859.523817:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048921344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762859.523821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762859.523834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762859.523836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762859.523840:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2309 00000100:00080000:0.0:1789762859.523842:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762859.523845:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762859.523849:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762859.523860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048921728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762859.523866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762859.523870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762859.523871:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762859.523875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2309 00000100:00080000:0.0:1789762859.523878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762859.523880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762859.523884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762859.523892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048921856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762859.523976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048921472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762859.524020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762859.524039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762859.524057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762859.779597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762860.419664:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762860.803582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762861.443576:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762861.827561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762862.467607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762862.595990:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789762862.596055:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762862.596079:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762862.596137:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762862.596148:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762862.596157:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762862.596166:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762862.596583:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762859/real 1789762862] req@ffff9811c2e2bb80 x1876700048921600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762862.596638:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048921600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762862.596654:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762862.596660:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762862.596665:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762862.596678:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762859/real 1789762862] req@ffff9811c2e29180 x1876700048921728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762862.596692:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048921728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762862.596700:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762862.596719:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762862.596723:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762862.596729:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762859/real 1789762862] req@ffff9811c2e2a680 x1876700048921856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762862.596747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048921856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762862.596760:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762862.596764:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762862.596767:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048921856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762862.596782:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762859/real 1789762862] req@ffff9811c2e2ad80 x1876700048921344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762862.596805:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048921344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762862.596814:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762862.596818:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762862.596822:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048921344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762862.596829:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762859/real 1789762862] req@ffff9811c2e29880 x1876700048921472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762862.596843:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048921472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762862.596852:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762862.596857:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762862.596861:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048921472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762862.851591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762863.491598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762863.875536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762864.515586:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762864.643572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762864.643581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762864.643584:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762864.643593:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2314 00000100:00080000:0.0:1789762864.643603:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762864.643609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762864.643641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048921984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762864.643664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762864.643668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762864.643670:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762864.643673:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2314 00000100:00080000:0.0:1789762864.643675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762864.643677:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762864.643684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762864.643714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048922112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762864.643725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762864.643729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762864.643731:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762864.643735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2314 00000100:00080000:0.0:1789762864.643738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762864.643748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762864.643748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048921984:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762864.643768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762864.643780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048922240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762864.643788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762864.643791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762864.643792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762864.643795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2314 00000100:00080000:0.0:1789762864.643797:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762864.643800:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762864.643804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762864.643812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048922368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762864.643818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762864.643822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762864.643823:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762864.643826:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2314 00000100:00080000:0.0:1789762864.643844:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762864.643854:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762864.643855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762864.643866:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762864.643883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048922496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762864.643925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762864/real 1789762864] req@ffff9811c2e28e00 x1876700048921984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762864.643950:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048921984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762864.643963:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762864.643968:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762864.643972:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048921984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762864.644009:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048922112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762864.644038:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762864.644052:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762864/real 1789762864] req@ffff9811c2e29c00 x1876700048922112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762864.644063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048922112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762864.644092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762864.644096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762864.644099:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048922112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762864.644106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789762864.644135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762864.644147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762864/real 1789762864] req@ffff9811c2e28700 x1876700048922240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762864.644159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048922240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762864.644166:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762864.644170:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762864.644173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789762864.644179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762864.644200:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762864.644212:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762864/real 1789762864] req@ffff9811c2e29880 x1876700048922368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762864.644223:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048922368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762864.644231:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762864.644234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762864.644245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789762864.644252:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789762864.644274:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762864.644285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762864/real 1789762864] req@ffff9811c2e2ad80 x1876700048922496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762864.644298:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048922496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762864.644306:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762864.644310:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789762864.644313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762864.899557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762865.539601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762865.923518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762866.563655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762866.947519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762867.587591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762867.971538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762868.611615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762868.995572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762869.635764:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762869.763665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762869.763682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762869.763689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762869.763705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2319 00000100:00080000:1.0:1789762869.763724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762869.763738:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762869.763809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048922624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762869.763857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762869.763863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762869.763867:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762869.763876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2319 00000100:00080000:1.0:1789762869.763885:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762869.763907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048922624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762869.763930:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762869.763951:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762869.763986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048922752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762869.763986:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762869.764020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762869.764026:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762869.764028:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762869.764037:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2319 00000100:00080000:1.0:1789762869.764046:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762869.764049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762869/real 1789762869] req@ffff9811c92c4000 x1876700048922624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762869.764058:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762869.764079:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762869.764096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048922880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762869.764108:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762869.764112:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762869.764114:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762869.764118:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2319 00000100:00080000:1.0:1789762869.764121:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762869.764125:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762869.764143:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762869.764158:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048923008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762869.764172:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762869.764174:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048922624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762869.764188:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762869.764190:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762869.764193:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762869.764201:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789762869.764212:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2319 00000100:00100000:3.0:1789762869.764212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048922624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762869.764235:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762869.764247:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762869.764260:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762869.764286:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048923136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762869.764407:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048922752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762869.764437:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762869.764453:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762869/real 1789762869] req@ffff9811c92c4e00 x1876700048922752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762869.764544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048922752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762869.764552:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762869.764560:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762869.764570:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048922752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762869.764578:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762869.764615:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762869.764670:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762869/real 1789762869] req@ffff9811c92c7480 x1876700048922880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762869.764680:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048922880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762869.764686:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762869.764689:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762869.764691:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048922880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762869.764696:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762869.764710:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762869.764718:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762869/real 1789762869] req@ffff9811c92c5c00 x1876700048923008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762869.764726:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048923008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762869.764731:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762869.764733:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762869.764735:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762869.764739:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762869.764749:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762869.764763:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762869/real 1789762869] req@ffff9811c92c6680 x1876700048923136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762869.764771:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048923136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762869.764777:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762869.764779:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762869.764781:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762870.019528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762870.659591:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762871.043536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762871.683594:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762872.067570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762872.707608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762873.091547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762873.731601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762874.115534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762874.755576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762874.883531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762874.883540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762874.883543:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762874.883553:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2324 00000100:00080000:1.0:1789762874.883562:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762874.883568:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762874.883620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048923264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762874.883649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762874.883652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762874.883654:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762874.883657:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2324 00000100:00080000:1.0:1789762874.883660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762874.883663:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762874.883671:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762874.883683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048923392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762874.883698:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048923264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762874.883717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762874.883720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762874.883721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762874.883725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2324 00000100:00080000:1.0:1789762874.883729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762874.883732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762874.883736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762874.883745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048923520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762874.883753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762874.883755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762874.883757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762874.883759:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2324 00000100:00080000:1.0:1789762874.883762:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762874.883764:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762874.883767:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762874.883773:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789762874.883776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048923648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762874.883784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762874.883787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762874.883789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762874.883792:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2324 00000100:00080000:1.0:1789762874.883800:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762874.883804:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762874.883809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762874.883814:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762874/real 1789762874] req@ffff9811c92c4380 x1876700048923264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762874.883820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048923776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762874.883839:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048923264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762874.883849:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762874.883852:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762874.883855:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048923264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762874.883874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048923392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762874.883889:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762874.883897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762874/real 1789762874] req@ffff9811c92c4a80 x1876700048923392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762874.883919:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048923392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762874.883925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762874.883927:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762874.883929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048923392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762874.883933:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762874.883945:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762874.883952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762874/real 1789762874] req@ffff9811c92c5f80 x1876700048923520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762874.883960:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048923520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762874.883965:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762874.883966:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762874.883968:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762874.883971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762874.883981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762874.883987:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762874/real 1789762874] req@ffff9811c92c6a00 x1876700048923648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762874.883993:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048923648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762874.884007:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762874.884009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762874.884011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762874.884015:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762874.884027:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762874.884033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762874/real 1789762874] req@ffff9811c92c6300 x1876700048923776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762874.884040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048923776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762874.884045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762874.884047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762874.884049:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048923776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762875.139530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762875.779579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762876.163591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762876.803581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762877.187570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762877.827578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762878.211540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762878.851557:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762879.235614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762879.875598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762880.003637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762880.003645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762880.003666:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762880.003675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2329 00000100:00080000:0.0:1789762880.003687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762880.003695:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762880.003731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048923904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762880.003760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762880.003763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762880.003764:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762880.003768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2329 00000100:00080000:0.0:1789762880.003771:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762880.003774:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762880.003782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762880.003794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048924032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762880.003804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762880.003808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762880.003811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762880.003818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2329 00000100:00100000:3.0:1789762880.003824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048923904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762880.003826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762880.003840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762880.003848:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762880.003866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048924160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762880.003887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762880.003890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762880.003892:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762880.003895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2329 00000100:00080000:0.0:1789762880.003899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762880.003903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762880.003908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762880.003920:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762880.003922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048924288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762880.003930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762880.003933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762880.003934:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762880.003938:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2329 00000100:00080000:0.0:1789762880.003943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762880.003946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762880.003950:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762880.003958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048924416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762880.003984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762880/real 1789762880] req@ffff9811c2e2a680 x1876700048923904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762880.004002:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048923904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762880.004012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762880.004027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762880.004030:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048923904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762880.004039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048924032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762880.004065:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762880.004079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762880/real 1789762880] req@ffff9811c2e29180 x1876700048924032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762880.004092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048924032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762880.004101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762880.004104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762880.004106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048924032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762880.004131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762880.004147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762880.004154:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762880/real 1789762880] req@ffff9811c2e2bb80 x1876700048924160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762880.004162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048924160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762880.004167:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762880.004169:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762880.004171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762880.004175:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762880.004195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762880.004201:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762880/real 1789762880] req@ffff9811c2e2b100 x1876700048924288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762880.004208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048924288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762880.004213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762880.004215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762880.004217:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762880.004220:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762880.004230:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762880.004236:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762880/real 1789762880] req@ffff9811c2e2aa00 x1876700048924416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762880.004243:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048924416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762880.004247:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762880.004249:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762880.004251:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762880.259552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762880.899607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762881.283614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762881.923662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762882.307578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762882.947619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762883.331593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762883.971642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762884.355595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762884.995634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762885.123599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762885.123614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762885.123620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762885.123634:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2334 00000100:00080000:0.0:1789762885.123662:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762885.123671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762885.123731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048924544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762885.123756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762885.123763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762885.123765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762885.123768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2334 00000100:00080000:0.0:1789762885.123774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762885.123778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762885.123788:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762885.123801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048924672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762885.123801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048924544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762885.123831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762885.123834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762885.123836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762885.123841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2334 00000100:00080000:0.0:1789762885.123843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762885.123845:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762885.123848:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762885.123855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048924800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762885.123861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762885.123870:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762885.123871:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762885.123875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2334 00000100:00080000:0.0:1789762885.123880:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762885.123883:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762885.123888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762885.123901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048924928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762885.123912:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762885.123915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762885.123918:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762885.123921:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762885.123924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2334 00000100:00080000:0.0:1789762885.123929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762885.123934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762885.123937:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762885.123947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048925056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762885.123964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762885/real 1789762885] req@ffff9811c2e28380 x1876700048924544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762885.124015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048924544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762885.124026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762885.124031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762885.124035:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048924544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762885.124062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048924672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762885.124083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762885.124092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762885/real 1789762885] req@ffff9811c2e28a80 x1876700048924672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762885.124101:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048924672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762885.124107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762885.124109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762885.124111:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048924672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762885.124115:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762885.124126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762885.124132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762885/real 1789762885] req@ffff9811c2e29f80 x1876700048924800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762885.124140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048924800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762885.124155:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762885.124157:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762885.124160:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762885.124164:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762885.124179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762885.124186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762885/real 1789762885] req@ffff9811c2e2b800 x1876700048924928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762885.124204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048924928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762885.124213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762885.124216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762885.124220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048924928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762885.124227:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762885.124250:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762885.124261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762885/real 1789762885] req@ffff9811c2e2b480 x1876700048925056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762885.124269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048925056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762885.124275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762885.124277:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762885.124285:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762885.379620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762886.019624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762886.403516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762887.043628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762887.427571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762888.067652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762888.451576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762889.091709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762889.475624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762890.115760:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762890.243686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762890.243701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762890.243708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762890.243723:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2339 00000100:00080000:0.0:1789762890.243740:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762890.243751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762890.243811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048925184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762890.243895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762890.243904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762890.243908:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762890.243920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2339 00000100:00080000:0.0:1789762890.243929:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762890.243964:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048925184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762890.243970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762890.243991:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762890.244018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048925312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762890.244038:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762890.244045:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762890.244054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762890.244058:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762890.244068:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2339 00000100:00080000:0.0:1789762890.244077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762890.244084:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762890.244091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762890.244105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762890/real 1789762890] req@ffff9811c2e2a300 x1876700048925184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762890.244107:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048925440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762890.244202:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762890.244206:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762890.244216:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762890.244216:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048925184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762890.244241:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2339 00000100:00080000:3.0:1789762890.244248:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762890.244256:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789762890.244259:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762890.244264:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048925184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762890.244272:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762890.244284:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762890.244295:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048925312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762890.244305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048925568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762890.244322:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762890.244333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762890.244339:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762890.244344:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762890.244353:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2339 00000100:00100000:3.0:1789762890.244355:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762890/real 1789762890] req@ffff9811c2e28000 x1876700048925312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762890.244361:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762890.244376:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762890.244386:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762890.244387:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048925312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762890.244405:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762890.244412:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048925696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762890.244412:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762890.244434:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048925312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762890.244443:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762890.244520:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762890.244531:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762890/real 1789762890] req@ffff9811c2e29500 x1876700048925440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762890.244544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048925440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762890.244557:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762890.244560:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762890.244567:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762890.244585:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762890.244605:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762890.244613:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762890/real 1789762890] req@ffff9811c2e2a300 x1876700048925568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762890.244621:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048925568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762890.244626:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762890.244628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762890.244630:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762890.244634:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762890.244644:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762890.244659:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762890/real 1789762890] req@ffff9811c2e2b480 x1876700048925696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762890.244667:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048925696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762890.244672:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762890.244675:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762890.244677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048925696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762890.499556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762891.139601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762891.523592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762892.163635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762892.547529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762893.187575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762893.571579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762894.211635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762894.595619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762895.235671:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762895.363623:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762895.363635:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762895.363641:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762895.363654:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2344 00000100:00080000:0.0:1789762895.363668:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762895.363678:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762895.363730:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048925824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762895.363764:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762895.363796:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762895.363799:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762895.363806:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2344 00000100:00080000:0.0:1789762895.363812:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762895.363818:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762895.363823:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048925824:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762895.363832:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762895.363850:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048925952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762895.363881:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762895.363885:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762895.363887:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762895.363891:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2344 00000100:00080000:0.0:1789762895.363894:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762895.363898:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762895.363903:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762895.363913:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048926080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762895.363923:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762895.363926:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762895.363927:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762895.363929:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762895.363933:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2344 00000100:00080000:0.0:1789762895.363936:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762895.363952:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762895.363966:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762895.363978:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048926208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762895.363984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762895/real 1789762895] req@ffff9811c2e2b800 x1876700048925824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762895.363990:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762895.363994:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762895.363996:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762895.364000:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2344 00000100:00080000:0.0:1789762895.364004:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762895.364005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048925824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762895.364008:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762895.364014:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762895.364019:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762895.364023:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048926336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762895.364024:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762895.364029:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048925824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762895.364059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048925952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762895.364088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762895.364114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762895/real 1789762895] req@ffff9811c2e29f80 x1876700048925952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762895.364124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048925952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762895.364131:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762895.364134:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762895.364137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048925952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762895.364142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762895.364159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762895.364168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762895/real 1789762895] req@ffff9811c2e28a80 x1876700048926080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762895.364177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048926080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762895.364182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762895.364184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762895.364186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762895.364190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762895.364203:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762895.364210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762895/real 1789762895] req@ffff9811c2e28380 x1876700048926208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762895.364228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048926208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762895.364241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762895.364243:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762895.364246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762895.364251:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762895.364268:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762895.364276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762895/real 1789762895] req@ffff9811c2e2aa00 x1876700048926336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762895.364285:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048926336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762895.364290:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762895.364292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762895.364294:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762895.619562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762896.259604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762896.643521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762897.283563:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762897.667522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762898.307568:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762898.691558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762899.331598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762899.715622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762900.355547:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762900.483543:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762900.483551:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762900.483555:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762900.483564:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2350 00000100:00080000:0.0:1789762900.483573:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762900.483579:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762900.483612:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048926464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762900.483633:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762900.483636:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762900.483638:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762900.483641:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2350 00000100:00080000:0.0:1789762900.483657:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762900.483661:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762900.483669:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762900.483679:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048926592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762900.483689:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762900.483692:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048926464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762900.483693:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762900.483696:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762900.483704:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2350 00000100:00080000:0.0:1789762900.483707:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762900.483709:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762900.483712:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762900.483721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048926720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762900.483729:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762900.483731:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762900.483733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762900.483735:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2350 00000100:00080000:0.0:1789762900.483738:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762900.483740:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762900.483743:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762900.483750:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048926848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762900.483759:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762900.483770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762900.483773:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762900.483775:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762900.483779:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2350 00000100:00080000:0.0:1789762900.483782:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762900.483784:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762900.483788:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762900.483798:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048926976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762900.483803:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762900/real 1789762900] req@ffff9811c2e2b100 x1876700048926464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762900.483828:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048926464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762900.483837:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762900.483840:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762900.483843:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048926464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762900.483861:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048926592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762900.483878:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762900.483886:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762900/real 1789762900] req@ffff9811c2e2bb80 x1876700048926592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762900.483895:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048926592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762900.483900:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762900.483903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762900.483905:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048926592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762900.483908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762900.483918:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762900.483924:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762900/real 1789762900] req@ffff9811c2e29180 x1876700048926720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762900.483931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048926720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762900.483946:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762900.483948:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762900.483951:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762900.483954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762900.483969:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762900.483975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762900/real 1789762900] req@ffff9811c2e2a680 x1876700048926848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762900.483983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048926848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762900.483988:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762900.483990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762900.483991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762900.483995:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762900.484004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762900.484010:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762900/real 1789762900] req@ffff9811c2e2ad80 x1876700048926976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762900.484017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048926976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762900.484022:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762900.484024:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762900.484030:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048926976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762900.739522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762901.379621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762901.763541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762902.403597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762902.787559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762903.427577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762903.811527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762904.451614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762904.835528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762905.475621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762905.603578:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762905.603590:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762905.603595:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762905.603607:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2355 00000100:00080000:0.0:1789762905.603619:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762905.603630:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762905.603668:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048927104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762905.603699:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762905.603703:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762905.603705:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762905.603710:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2355 00000100:00080000:0.0:1789762905.603713:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762905.603716:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762905.603738:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762905.603755:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048927232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762905.603770:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048927104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762905.603774:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762905.603778:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762905.603780:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762905.603798:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2355 00000100:00080000:0.0:1789762905.603803:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762905.603808:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762905.603813:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762905.603833:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048927360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762905.603846:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762905.603851:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762905.603854:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762905.603858:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2355 00000100:00080000:0.0:1789762905.603866:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762905.603872:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762905.603878:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762905.603894:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048927488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762905.603901:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762905.603916:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762905.603919:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762905.603927:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762905.603935:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2355 00000100:00080000:0.0:1789762905.603942:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762905.603949:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762905.603957:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762905.603968:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762905/real 1789762905] req@ffff9811c2e29880 x1876700048927104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762905.603978:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048927616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762905.604012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048927104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762905.604026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762905.604032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762905.604036:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048927104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762905.604067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048927232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762905.604094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762905.604107:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762905/real 1789762905] req@ffff9811c2e28700 x1876700048927232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762905.604119:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048927232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762905.604128:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762905.604131:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762905.604135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048927232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762905.604141:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762905.604159:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762905.604170:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762905/real 1789762905] req@ffff9811c2e29c00 x1876700048927360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762905.604199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048927360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762905.604207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762905.604212:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762905.604216:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762905.604223:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762905.604247:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762905.604267:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762905/real 1789762905] req@ffff9811c2e28e00 x1876700048927488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762905.604281:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048927488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762905.604295:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762905.604311:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762905.604319:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762905.604331:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762905.604357:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762905.604368:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762905/real 1789762905] req@ffff9811c2e2ad80 x1876700048927616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762905.604377:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048927616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762905.604418:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762905.604422:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762905.604425:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048927616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762905.859569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762906.499628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762906.883588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762907.523602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762907.907582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762908.547582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762908.931580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762909.571573:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762909.955551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762910.595566:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762910.723536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762910.723545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762910.723548:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762910.723557:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2360 00000100:00080000:0.0:1789762910.723566:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762910.723572:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762910.723616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048927744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762910.723639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762910.723642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762910.723643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762910.723646:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2360 00000100:00080000:0.0:1789762910.723648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762910.723650:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762910.723657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762910.723666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048927872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762910.723685:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048927744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762910.723702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762910.723705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762910.723706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762910.723711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2360 00000100:00080000:0.0:1789762910.723714:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762910.723717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762910.723721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762910.723731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048928000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762910.723738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762910.723742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762910.723743:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762910.723745:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2360 00000100:00080000:0.0:1789762910.723748:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762910.723751:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789762910.723755:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762910.723760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762910.723768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048928128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762910.723776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762910.723778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762910.723780:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762910.723783:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2360 00000100:00080000:0.0:1789762910.723790:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762910.723792:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762910/real 1789762910] req@ffff9811c2e2a680 x1876700048927744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762910.723797:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762910.723806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762910.723817:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048927744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762910.723818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048928256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762910.723838:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762910.723843:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762910.723847:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048927744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762910.723866:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048927872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762910.723883:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762910.723891:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762910/real 1789762910] req@ffff9811c2e29180 x1876700048927872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762910.723899:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048927872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762910.723905:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762910.723914:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762910.723917:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048927872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762910.723923:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762910.723945:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762910.723965:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762910/real 1789762910] req@ffff9811c2e2bb80 x1876700048928000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762910.723979:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048928000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762910.723987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762910.723990:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762910.723994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762910.724000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762910.724024:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762910.724036:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762910/real 1789762910] req@ffff9811c2e2b100 x1876700048928128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762910.724047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048928128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762910.724053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762910.724055:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762910.724058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762910.724062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762910.724088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762910.724100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762910/real 1789762910] req@ffff9811c2e2aa00 x1876700048928256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762910.724114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048928256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762910.724123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762910.724127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762910.724130:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762910.979558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762911.619651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762912.003535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762912.643581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762913.027551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762913.667542:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762914.051559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762914.691639:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762915.075529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789762915.715668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762915.843589:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762915.843602:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762915.843607:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762915.843620:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2365 00000100:00080000:2.0:1789762915.843635:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762915.843644:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762915.843735:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048928384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762915.843812:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762915.843819:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762915.843822:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762915.843829:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2365 00000100:00080000:2.0:1789762915.843837:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762915.843871:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762915.843887:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762915.843915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048928384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762915.843923:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048928512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762915.843938:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762915.843942:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762915.843943:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762915.843947:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2365 00000100:00080000:2.0:1789762915.843952:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762915.843955:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762915.843960:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762915.843971:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048928640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762915.843980:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762915.843983:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762915.843985:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762915.843988:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2365 00000100:00080000:2.0:1789762915.843992:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762915.843995:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762915.843999:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762915.844005:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762915.844011:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048928768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762915.844036:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762915.844052:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762915.844056:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762915.844063:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2365 00000100:00080000:2.0:1789762915.844070:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762915.844077:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762915.844086:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762915.844094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762915/real 1789762915] req@ffff9811d863b800 x1876700048928384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762915.844127:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048928896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762915.844135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048928384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762915.844164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762915.844180:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762915.844187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048928384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762915.844220:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048928512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762915.844257:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762915.844279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762915/real 1789762915] req@ffff9811d863a300 x1876700048928512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762915.844301:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048928512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762915.844337:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762915.844340:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762915.844345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048928512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762915.844352:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762915.844392:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762915.844404:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762915/real 1789762915] req@ffff9811d863b480 x1876700048928640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762915.844414:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048928640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762915.844421:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762915.844424:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762915.844427:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762915.844431:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762915.844476:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762915.844486:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762915/real 1789762915] req@ffff9811d863aa00 x1876700048928768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762915.844533:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048928768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762915.844545:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762915.844549:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762915.844554:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762915.844563:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762915.844595:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762915.844612:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762915/real 1789762915] req@ffff9811d8638000 x1876700048928896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762915.844632:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048928896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762915.844644:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762915.844649:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762915.844670:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048928896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762916.099574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762916.739656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762917.123600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762917.763634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762918.147616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762918.787611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762919.171561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762919.811590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762920.195572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762920.835616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762920.963629:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762920.963645:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762920.963652:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762920.963666:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2370 00000100:00080000:0.0:1789762920.963683:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762920.963711:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762920.963774:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048929024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762920.963813:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762920.963817:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762920.963820:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762920.963824:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2370 00000100:00080000:0.0:1789762920.963829:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762920.963833:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762920.963847:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762920.963861:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048929152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762920.963876:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762920.963884:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048929024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762920.963924:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762920.963926:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762920.963932:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2370 00000100:00080000:0.0:1789762920.963937:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762920.963942:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762920.963948:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762920.963962:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048929280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762920.963987:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762920.963990:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762920.963992:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762920.963995:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2370 00000100:00080000:0.0:1789762920.963998:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762920.964001:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762920.964006:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762920.964012:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789762920.964018:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048929408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762920.964038:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762920.964042:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762920.964043:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762920.964047:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2370 00000100:00080000:0.0:1789762920.964050:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762920.964054:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762920.964066:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762920.964074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762920/real 1789762920] req@ffff9811c2e28380 x1876700048929024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762920.964079:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048929536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762920.964153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048929024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762920.964169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762920.964174:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762920.964179:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048929024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762920.964224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048929152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762920.964242:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762920.964252:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762920/real 1789762920] req@ffff9811c2e28a80 x1876700048929152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762920.964260:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048929152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762920.964266:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762920.964268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762920.964287:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048929152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762920.964291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762920.964306:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762920.964314:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762920/real 1789762920] req@ffff9811c2e29f80 x1876700048929280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762920.964328:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048929280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762920.964334:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762920.964336:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762920.964338:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762920.964342:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762920.964355:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762920.964361:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762920/real 1789762920] req@ffff9811c2e2b800 x1876700048929408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762920.964368:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048929408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762920.964373:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762920.964375:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762920.964376:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762920.964379:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762920.964389:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762920.964395:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762920/real 1789762920] req@ffff9811c2e2b480 x1876700048929536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762920.964401:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048929536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762920.964414:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762920.964416:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762920.964418:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762921.219595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762921.859611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762922.243552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762922.883736:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762923.267593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762923.907571:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762924.291533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762924.931643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762925.315591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762925.955577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762926.083586:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762926.083598:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762926.083602:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762926.083611:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2375 00000100:00080000:0.0:1789762926.083620:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762926.083627:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762926.083662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048929664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762926.083684:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762926.083687:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762926.083688:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762926.083694:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2375 00000100:00080000:0.0:1789762926.083697:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762926.083699:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762926.083708:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762926.083721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048929792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762926.083734:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762926.083739:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048929664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789762926.083740:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762926.083745:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762926.083752:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2375 00000100:00080000:0.0:1789762926.083772:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762926.083776:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762926.083781:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762926.083792:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048929920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762926.083800:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762926.083810:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762926.083812:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762926.083815:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2375 00000100:00080000:0.0:1789762926.083818:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762926.083821:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762926.083825:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762926.083835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048930048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762926.083842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048929792:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789762926.083843:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762926.083850:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762926.083852:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762926.083856:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2375 00000100:00080000:0.0:1789762926.083859:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762926.083863:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762926.083868:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762926.083876:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762926.083882:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048930176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762926.083910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762926.083949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762926.339518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762926.979625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762927.363533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762928.003620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762928.387548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762929.027605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762929.156015:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789762929.156118:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762929.156168:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762929.156255:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762929.156274:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789762929.156291:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789762929.156356:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762926/real 1789762929] req@ffff9811c2e29500 x1876700048929792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789762929.156412:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789762929.156439:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048929792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762929.156542:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762929.156554:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762929.156562:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048929792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762929.156576:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762926/real 1789762929] req@ffff9811c2e28000 x1876700048929920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762929.156600:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048929920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762929.156612:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762929.156618:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762929.156624:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048929920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762929.156634:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762926/real 1789762929] req@ffff9811c2e2a300 x1876700048929664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762929.156653:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048929664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762929.156664:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762929.156669:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762929.156674:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048929664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762929.156716:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762926/real 1789762929] req@ffff9811c2e2b480 x1876700048930048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762929.156739:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048930048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762929.156765:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762929.156772:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762929.156778:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762929.156790:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762926/real 1789762929] req@ffff9811c2e2b800 x1876700048930176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762929.156811:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048930176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762929.156822:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762929.156827:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762929.156833:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762929.411552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762930.051639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762930.435559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762931.075614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762931.203535:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762931.203544:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762931.203548:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762931.203556:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2380 00000100:00080000:0.0:1789762931.203566:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762931.203573:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762931.203603:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048930304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762931.203627:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762931.203630:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762931.203631:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762931.203634:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2380 00000100:00080000:0.0:1789762931.203636:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762931.203645:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762931.203653:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762931.203662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048930432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762931.203671:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762931.203674:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762931.203675:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762931.203680:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2380 00000100:00080000:0.0:1789762931.203683:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762931.203691:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762931.203692:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048930304:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789762931.203715:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762931.203731:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048930560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762931.203742:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762931.203746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762931.203749:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762931.203754:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2380 00000100:00080000:0.0:1789762931.203759:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762931.203766:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789762931.203768:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762931.203775:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762931.203791:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048930688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762931.203802:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762931.203805:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762931.203807:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762931.203812:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2380 00000100:00100000:3.0:1789762931.203817:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762931/real 1789762931] req@ffff9811c2e29f80 x1876700048930304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762931.203820:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762931.203841:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762931.203847:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048930304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762931.203850:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762931.203866:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762931.203872:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048930816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762931.203876:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762931.203884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048930304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762931.203904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048930432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762931.203940:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762931.203955:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762931/real 1789762931] req@ffff9811c2e28a80 x1876700048930432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762931.203969:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048930432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762931.203977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762931.203981:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762931.203985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048930432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762931.204010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762931.204032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762931.204044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762931/real 1789762931] req@ffff9811c2e28380 x1876700048930560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762931.204056:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048930560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762931.204064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762931.204067:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762931.204071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762931.204077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762931.204097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762931.204117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762931/real 1789762931] req@ffff9811c2e2aa00 x1876700048930688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762931.204140:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048930688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762931.204149:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762931.204153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762931.204157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762931.204164:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762931.204188:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762931.204220:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762931/real 1789762931] req@ffff9811c2e2b100 x1876700048930816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762931.204234:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048930816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762931.204243:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762931.204247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762931.204250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048930816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762931.459589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762932.099727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762932.483528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762933.123606:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762933.507514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762934.147618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762934.531537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762935.171626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762935.555525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762936.195597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762936.323618:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762936.323630:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762936.323635:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762936.323647:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2385 00000100:00080000:2.0:1789762936.323659:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762936.323667:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762936.323714:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048930944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762936.323744:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762936.323749:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762936.323751:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762936.323760:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2385 00000100:00080000:2.0:1789762936.323764:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762936.323768:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762936.323781:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762936.323795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048931072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762936.323810:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048930944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762936.323834:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762936.323838:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762936.323839:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762936.323844:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2385 00000100:00080000:2.0:1789762936.323848:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762936.323852:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762936.323856:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762936.323868:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048931200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762936.323875:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762936.323878:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762936.323880:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762936.323883:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2385 00000100:00080000:2.0:1789762936.323886:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762936.323890:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762936.323894:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789762936.323895:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762936.323903:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048931328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762936.323910:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762936.323913:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762936.323915:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762936.323917:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2385 00000100:00080000:2.0:1789762936.323926:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762936.323930:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762936.323934:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762936.323944:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048931456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762936.323952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762936/real 1789762936] req@ffff9811d8638700 x1876700048930944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762936.323977:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048930944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762936.323989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762936.323994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762936.323999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048930944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762936.324027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048931072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762936.324047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762936.324057:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762936/real 1789762936] req@ffff9811d8638e00 x1876700048931072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762936.324065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048931072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762936.324071:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762936.324073:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762936.324075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048931072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762936.324078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762936.324088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762936.324094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762936/real 1789762936] req@ffff9811d8638380 x1876700048931200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762936.324101:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048931200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762936.324115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762936.324117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762936.324120:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762936.324124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762936.324138:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762936.324144:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762936/real 1789762936] req@ffff9811d8638000 x1876700048931328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762936.324155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048931328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762936.324177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762936.324181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762936.324184:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762936.324190:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762936.324219:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762936.324231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762936/real 1789762936] req@ffff9811d863aa00 x1876700048931456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762936.324240:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048931456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762936.324245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762936.324247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762936.324254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762936.579524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762937.219603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762937.603511:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762938.243717:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762938.627563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762939.267583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762939.651530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762940.291598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762940.675527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762941.315686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762941.443624:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762941.443639:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762941.443645:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762941.443660:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2391 00000100:00080000:2.0:1789762941.443702:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762941.443712:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762941.443788:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048931584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762941.443838:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762941.443846:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762941.443849:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762941.443858:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2391 00000100:00080000:2.0:1789762941.443865:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762941.443874:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762941.443893:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762941.443916:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048931584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762941.443924:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048931712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762941.443954:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762941.443988:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1789762941.443995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762941.443996:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762941.444010:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2391 00000100:00080000:2.0:1789762941.444020:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762941.444027:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762941.444037:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762941.444053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762941/real 1789762941] req@ffff9811d863b480 x1876700048931584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762941.444059:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048931840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762941.444144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048931584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762941.444159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762941.444160:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762941.444178:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762941.444178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789762941.444190:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762941.444192:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048931584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762941.444206:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2391 00000100:00080000:2.0:1789762941.444212:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762941.444220:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762941.444229:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762941.444234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048931712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762941.444255:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048931968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762941.444271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789762941.444286:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762941.444291:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762941.444293:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762941.444299:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762941/real 1789762941] req@ffff9811d863a300 x1876700048931712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762941.444305:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2391 00000100:00080000:2.0:1789762941.444317:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762941.444332:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762941.444336:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048931712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762941.444358:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789762941.444362:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762941.444363:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762941.444384:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048932096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762941.444389:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048931712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762941.444418:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762941.444451:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762941.444496:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762941/real 1789762941] req@ffff9811d863b800 x1876700048931840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762941.444507:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048931840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762941.444518:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762941.444521:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762941.444525:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762941.444541:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762941.444559:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762941.444569:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762941/real 1789762941] req@ffff9811d8638a80 x1876700048931968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762941.444578:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048931968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762941.444596:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762941.444601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762941.444605:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048931968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762941.444614:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762941.444645:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762941.444670:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762941/real 1789762941] req@ffff9811d863a680 x1876700048932096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762941.444679:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048932096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762941.444685:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762941.444688:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762941.444690:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762941.699552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762942.339598:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762942.723594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762943.363614:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762943.747529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762944.387647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762944.771574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762945.411585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762945.795522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762946.435701:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762946.563564:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762946.563580:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762946.563586:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762946.563595:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2396 00000100:00080000:1.0:1789762946.563605:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762946.563612:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762946.563646:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048932224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762946.563674:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762946.563679:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762946.563681:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762946.563685:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2396 00000100:00080000:1.0:1789762946.563689:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762946.563694:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762946.563718:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048932224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789762946.563720:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762946.563736:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048932352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762946.563747:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762946.563750:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762946.563752:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762946.563755:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2396 00000100:00080000:1.0:1789762946.563759:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762946.563762:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762946.563766:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762946.563775:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048932480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762946.563779:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762946.563788:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762946.563791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762946.563793:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762946.563797:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2396 00000100:00080000:1.0:1789762946.563799:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762946.563802:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762946.563806:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762946.563815:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048932608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762946.563820:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762946/real 1789762946] req@ffff9811c92c7100 x1876700048932224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762946.563827:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762946.563835:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762946.563842:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762946.563846:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048932224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762946.563849:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2396 00000100:00080000:1.0:1789762946.563855:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762946.563861:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789762946.563863:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762946.563867:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762946.563869:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762946.563874:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048932224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762946.563879:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048932736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762946.563909:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048932352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762946.563928:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762946.563937:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762946/real 1789762946] req@ffff9811c92c7800 x1876700048932352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762946.563946:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048932352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762946.563952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762946.563955:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762946.563957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048932352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762946.563961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762946.563972:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762946.563978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762946/real 1789762946] req@ffff9811c92c4700 x1876700048932480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762946.563986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048932480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762946.563991:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762946.563993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762946.563995:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762946.563998:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762946.564008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762946.564023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762946/real 1789762946] req@ffff9811c92c6300 x1876700048932608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762946.564037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048932608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762946.564043:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762946.564045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762946.564047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762946.564051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762946.564065:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762946.564071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762946/real 1789762946] req@ffff9811c92c6a00 x1876700048932736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762946.564079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048932736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762946.564098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762946.564101:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762946.564103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048932736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762946.819591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762947.459599:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762947.843537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762948.483617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762948.867548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762949.507585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762949.891624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762950.531624:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762950.915549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762951.555663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762951.683527:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762951.683538:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762951.683542:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762951.683553:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2401 00000100:00080000:3.0:1789762951.683564:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762951.683571:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762951.683611:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048932864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762951.683645:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762951.683648:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762951.683650:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762951.683652:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2401 00000100:00080000:3.0:1789762951.683655:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762951.683657:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762951.683666:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762951.683678:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048932992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762951.683697:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762951.683700:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762951.683701:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762951.683703:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2401 00000100:00080000:3.0:1789762951.683705:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762951.683708:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762951.683710:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762951.683730:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048933120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762951.683736:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762951.683738:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762951.683739:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762951.683741:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2401 00000100:00080000:3.0:1789762951.683744:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762951.683747:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762951.683761:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762951.683767:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048933248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762951.683775:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762951.683777:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762951.683778:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762951.683783:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2401 00000100:00100000:2.0:1789762951.683784:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048932864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762951.683785:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762951.683791:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762951.683795:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762951.683800:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048933376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762951.683892:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762951.683933:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762951/real 1789762951] req@ffff9811e1a23100 x1876700048932864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762951.683957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048932864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762951.683969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762951.683984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762951.683989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048932864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762951.683997:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048932992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762951.684035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762951.684067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762951/real 1789762951] req@ffff9811e1a23800 x1876700048932992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762951.684082:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048932992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762951.684092:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762951.684096:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762951.684099:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048932992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762951.684106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762951.684135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762951.684144:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762951/real 1789762951] req@ffff9811e1a20380 x1876700048933120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762951.684153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048933120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762951.684158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762951.684161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762951.684163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762951.684186:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762951.684199:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762951.684205:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762951/real 1789762951] req@ffff9811e1a20e00 x1876700048933248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762951.684213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048933248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762951.684218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762951.684220:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762951.684222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762951.684226:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762951.684243:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762951.684254:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762951/real 1789762951] req@ffff9811e1a20000 x1876700048933376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762951.684291:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048933376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762951.684308:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762951.684311:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762951.684316:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762951.939570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762952.579708:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762952.963586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762953.603622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762953.987534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762954.627637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762955.011551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762955.651687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762956.035642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762956.675715:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762956.803612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762956.803620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762956.803624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762956.803632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2406 00000100:00080000:2.0:1789762956.803642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762956.803648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762956.803681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048933504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762956.803709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762956.803712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762956.803713:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762956.803716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2406 00000100:00080000:2.0:1789762956.803718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762956.803720:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762956.803727:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762956.803735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048933632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762956.803764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762956.803769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762956.803774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762956.803775:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048933504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762956.803780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2406 00000100:00080000:2.0:1789762956.803790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762956.803796:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762956.803801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762956.803811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048933760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762956.803818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762956.803820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762956.803822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762956.803824:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2406 00000100:00080000:2.0:1789762956.803826:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762956.803829:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762956.803832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762956.803839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048933888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762956.803845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762956.803849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762956.803850:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762956.803852:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2406 00000100:00080000:2.0:1789762956.803859:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762956.803864:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762956.803867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762956.803884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048934016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762956.803886:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762956.803962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762956/real 1789762956] req@ffff9811d8639f80 x1876700048933504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762956.803985:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048933504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762956.803999:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762956.804004:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762956.804025:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048933504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762956.804057:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048933632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762956.804085:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762956.804099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762956/real 1789762956] req@ffff9811d8639500 x1876700048933632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762956.804113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048933632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762956.804123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762956.804127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762956.804131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048933632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762956.804138:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762956.804177:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762956.804190:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762956/real 1789762956] req@ffff9811d8639880 x1876700048933760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762956.804203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048933760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762956.804212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762956.804216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762956.804220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762956.804228:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762956.804249:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762956.804261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762956/real 1789762956] req@ffff9811d863b100 x1876700048933888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762956.804274:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048933888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762956.804282:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762956.804286:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762956.804290:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048933888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762956.804296:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762956.804315:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762956.804325:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762956/real 1789762956] req@ffff9811d8639c00 x1876700048934016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762956.804353:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048934016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762956.804362:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762956.804365:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762956.804368:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762957.059595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762957.699620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762958.083644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762958.723585:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762959.107528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762959.747650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762960.131601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762960.771620:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762961.155575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762961.795658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762961.923585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762961.923596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762961.923617:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762961.923628:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2411 00000100:00080000:0.0:1789762961.923639:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762961.923647:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762961.923682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048934144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762961.923713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762961.923718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762961.923720:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762961.923725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2411 00000100:00080000:0.0:1789762961.923728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762961.923732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762961.923741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762961.923754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048934272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762961.923765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762961.923769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762961.923771:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762961.923775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2411 00000100:00080000:0.0:1789762961.923779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762961.923784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762961.923789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762961.923794:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048934144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762961.923802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048934400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762961.923828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762961.923832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762961.923834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762961.923839:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2411 00000100:00080000:0.0:1789762961.923845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762961.923850:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789762961.923853:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789762961.923861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762961.923877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048934528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762961.923888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762961.923893:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789762961.923894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762961/real 1789762961] req@ffff9811c2e2bb80 x1876700048934144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762961.923896:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762961.923906:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2411 00000100:00080000:0.0:1789762961.923912:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762961.923918:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048934144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762961.923919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762961.923928:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789762961.923931:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762961.923936:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762961.923940:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048934144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762961.923945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048934656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762961.923949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048934272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762961.923995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762961.924020:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762961/real 1789762961] req@ffff9811c2e29180 x1876700048934272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762961.924035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048934272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762961.924044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762961.924049:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762961.924052:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048934272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762961.924072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762961.924088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762961.924096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762961/real 1789762961] req@ffff9811c2e2a680 x1876700048934400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762961.924108:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048934400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762961.924116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762961.924120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762961.924123:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762961.924130:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762961.924152:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762961.924163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762961/real 1789762961] req@ffff9811c2e2ad80 x1876700048934528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762961.924177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048934528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762961.924194:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762961.924198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762961.924202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762961.924206:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762961.924224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762961.924232:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762961/real 1789762961] req@ffff9811c2e28e00 x1876700048934656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762961.924239:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048934656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762961.924245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762961.924247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762961.924249:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048934656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762962.179568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762962.819600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762963.203527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762963.843629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762964.227580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762964.867592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762965.251566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762965.891655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762966.275537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762966.915698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762967.043670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762967.043689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762967.043695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762967.043711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2416 00000100:00080000:2.0:1789762967.043729:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762967.043740:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762967.043802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048934784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762967.043845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762967.043850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762967.043852:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762967.043857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2416 00000100:00080000:2.0:1789762967.043862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762967.043866:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762967.043880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762967.043898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048934912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762967.043915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762967.043941:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048934784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789762967.043977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762967.043982:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762967.043990:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2416 00000100:00080000:2.0:1789762967.044008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762967.044020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789762967.044029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789762967.044035:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762967.044059:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048935040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762967.044074:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762967.044079:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762967.044082:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762967.044089:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2416 00000100:00080000:2.0:1789762967.044095:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762967.044096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762967/real 1789762967] req@ffff9811d8639180 x1876700048934784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762967.044112:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789762967.044131:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762967.044151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048935168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762967.044197:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762967.044207:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789762967.044211:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762967.044217:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2416 00000100:00080000:2.0:1789762967.044223:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762967.044225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048934784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762967.044238:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789762967.044245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762967.044252:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762967.044268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048934784:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789762967.044273:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762967.044299:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048934912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762967.044304:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048935296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762967.044351:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762967.044368:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762967/real 1789762967] req@ffff9811d863bb80 x1876700048934912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762967.044385:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048934912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762967.044392:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762967.044395:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762967.044398:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048934912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762967.044404:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762967.044419:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762967.044426:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762967/real 1789762967] req@ffff9811d863ad80 x1876700048935040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762967.044434:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048935040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762967.044450:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762967.044453:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762967.044456:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762967.044498:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762967.044517:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762967.044526:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762967/real 1789762967] req@ffff9811d8639c00 x1876700048935168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762967.044534:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048935168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762967.044540:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762967.044542:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762967.044544:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762967.044557:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762967.044569:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762967.044575:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762967/real 1789762967] req@ffff9811d863b100 x1876700048935296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762967.044583:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048935296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762967.044588:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762967.044590:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762967.044599:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762967.299576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762967.939712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762968.323594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762968.963689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762969.347574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762969.987731:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762970.371564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762971.011661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762971.395553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762972.035620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762972.163544:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762972.163553:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762972.163556:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762972.163565:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2421 00000100:00080000:1.0:1789762972.163575:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762972.163581:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762972.163627:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048935424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762972.163649:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762972.163652:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762972.163653:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762972.163656:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2421 00000100:00080000:1.0:1789762972.163659:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762972.163661:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762972.163669:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762972.163678:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048935552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762972.163686:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762972.163689:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762972.163692:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762972.163694:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048935424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789762972.163697:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2421 00000100:00080000:1.0:1789762972.163701:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762972.163706:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762972.163708:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762972.163715:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048935680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762972.163722:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762972.163724:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762972.163725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762972.163732:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2421 00000100:00080000:1.0:1789762972.163736:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762972.163739:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762972.163743:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762972.163752:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048935808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762972.163759:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762972.163760:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762972.163766:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762972.163767:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762972.163771:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2421 00000100:00080000:1.0:1789762972.163774:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762972.163777:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762972.163780:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762972.163788:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048935936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762972.163803:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762972/real 1789762972] req@ffff9811c92c5f80 x1876700048935424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762972.163818:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048935424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762972.163827:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762972.163830:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762972.163833:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048935424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762972.163850:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048935552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762972.163865:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762972.163873:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762972/real 1789762972] req@ffff9811c92c4a80 x1876700048935552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762972.163881:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048935552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762972.163887:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762972.163889:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762972.163914:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048935552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762972.163919:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789762972.163935:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762972.163943:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762972/real 1789762972] req@ffff9811c92c4380 x1876700048935680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762972.163951:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048935680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762972.163957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762972.163959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762972.163961:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789762972.163964:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762972.163975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762972.163981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762972/real 1789762972] req@ffff9811c92c6680 x1876700048935808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762972.163988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048935808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762972.163993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762972.163995:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762972.163997:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762972.164000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789762972.164010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762972.164023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762972/real 1789762972] req@ffff9811c92c5c00 x1876700048935936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762972.164031:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048935936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762972.164036:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762972.164038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789762972.164040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048935936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762972.419536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762973.059599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762973.443568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762974.083631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762974.467638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762975.107639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762975.491635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762976.131702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762976.515600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762977.155667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762977.283590:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762977.283602:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762977.283607:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762977.283625:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2426 00000100:00080000:3.0:1789762977.283636:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762977.283644:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762977.283684:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048936064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762977.283731:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762977.283735:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762977.283748:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762977.283757:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2426 00000100:00080000:3.0:1789762977.283764:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762977.283769:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762977.283782:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762977.283792:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048936064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762977.283824:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048936192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762977.283840:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762977.283844:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762977.283847:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762977.283852:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2426 00000100:00080000:3.0:1789762977.283881:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762977.283903:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789762977.283910:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789762977.283915:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762977.283939:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048936320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762977.283956:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762977.283963:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762977.283966:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762977.283971:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2426 00000100:00080000:3.0:1789762977.283976:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762977.283978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762977/real 1789762977] req@ffff9811e1a20a80 x1876700048936064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762977.283984:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762977.283995:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762977.284018:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048936448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762977.284019:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048936064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762977.284037:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762977.284041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762977.284045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048936064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762977.284066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048936192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762977.284088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762977.284098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762977/real 1789762977] req@ffff9811e1a21f80 x1876700048936192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762977.284109:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048936192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762977.284116:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762977.284119:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762977.284122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048936192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762977.284128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762977.284160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762977.284168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762977/real 1789762977] req@ffff9811e1a20000 x1876700048936320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762977.284176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048936320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762977.284182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762977.284184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762977.284187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936320:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789762977.284250:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762977.284253:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762977.284255:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762977.284260:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2426 00000100:00080000:3.0:1789762977.284265:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762977.284273:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762977.284287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762977.284312:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762977.284335:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762977/real 1789762977] req@ffff9811e1a20e00 x1876700048936448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762977.284340:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762977.284352:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048936448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762977.284360:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048936576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762977.284377:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762977.284382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762977.284387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762977.284402:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762977.284416:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762977.284423:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762977/real 1789762977] req@ffff9811e1a20380 x1876700048936576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762977.284442:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048936576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762977.284448:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762977.284450:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762977.284452:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762977.539590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762978.179662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762978.563535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762979.203622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762979.587534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762980.227648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762980.611568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762981.251700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762981.635574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762982.275672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762982.403567:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762982.403580:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762982.403585:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762982.403600:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2431 00000100:00080000:3.0:1789762982.403613:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762982.403623:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789762982.403670:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048936704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762982.403735:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762982.403741:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762982.403743:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762982.403750:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2431 00000100:00080000:3.0:1789762982.403758:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762982.403767:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762982.403782:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762982.403809:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048936832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762982.403855:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762982.403864:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762982.403868:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762982.403877:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2431 00000100:00100000:2.0:1789762982.403886:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048936704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762982.403902:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762982.403912:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762982.403923:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762982.403946:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048936960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762982.403964:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762982.403969:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762982.403972:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762982.403979:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2431 00000800:00000100:2.0:1789762982.403984:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789762982.404000:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762982.404009:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789762982.404019:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789762982.404045:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048937088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762982.404053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762982/real 1789762982] req@ffff9811e1a23800 x1876700048936704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762982.404097:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762982.404105:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048936704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762982.404109:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789762982.404119:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762982.404132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762982.404135:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2431 00000100:00080000:2.0:1789762982.404145:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789762982.404148:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762982.404156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048936704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789762982.404165:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762982.404175:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048936832:192.168.203.113@tcp:38:kworker.0 00000008:00100000:3.0:1789762982.404178:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789762982.404224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789762982.404225:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048937216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762982.404250:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762982/real 1789762982] req@ffff9811e1a23100 x1876700048936832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762982.404272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048936832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762982.404307:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762982.404313:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762982.404318:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048936832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762982.404347:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762982.404375:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762982.404389:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762982/real 1789762982] req@ffff9811e1a20700 x1876700048936960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762982.404404:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048936960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762982.404414:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762982.404418:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762982.404422:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048936960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762982.404429:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762982.404447:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762982.404457:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762982/real 1789762982] req@ffff9811e1a21500 x1876700048937088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762982.404520:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048937088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762982.404531:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762982.404535:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762982.404549:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762982.404558:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762982.404588:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762982.404602:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762982/real 1789762982] req@ffff9811e1a23480 x1876700048937216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762982.404618:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048937216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762982.404628:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762982.404632:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762982.404635:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762982.659587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762983.299651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762983.683554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762984.323734:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762984.707578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762985.347678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762985.731562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762986.371600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762986.755541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762987.395610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762987.523583:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762987.523599:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762987.523634:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762987.523646:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2437 00000100:00080000:1.0:1789762987.523659:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762987.523667:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762987.523739:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048937344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762987.523768:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762987.523802:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762987.523805:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762987.523809:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2437 00000100:00080000:1.0:1789762987.523813:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762987.523817:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762987.523853:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762987.523856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048937344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762987.523866:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048937472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762987.523876:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762987.523879:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762987.523881:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762987.523884:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2437 00000100:00080000:1.0:1789762987.523887:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762987.523889:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762987.523893:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762987.523901:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048937600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762987.523917:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762987.523919:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762987.523921:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762987.523923:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2437 00000100:00080000:1.0:1789762987.523926:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762987.523930:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762987.523934:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762987.523940:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762987.523949:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048937728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762987.523955:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762987.523958:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762987.523959:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762987.523962:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2437 00000100:00080000:1.0:1789762987.523964:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762987.523967:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762987.523970:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762987.523975:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762987/real 1789762987] req@ffff9811c92c7480 x1876700048937344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762987.523979:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048937856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762987.524008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048937344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762987.524018:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762987.524022:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762987.524024:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048937344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789762987.524046:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048937472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762987.524063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762987.524080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762987/real 1789762987] req@ffff9811c92c4e00 x1876700048937472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762987.524097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048937472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762987.524103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762987.524105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762987.524107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048937472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762987.524111:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762987.524123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762987.524129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762987/real 1789762987] req@ffff9811c92c4000 x1876700048937600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762987.524136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048937600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762987.524141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762987.524143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762987.524145:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762987.524148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762987.524175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762987.524181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762987/real 1789762987] req@ffff9811c92c5500 x1876700048937728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762987.524188:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048937728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762987.524198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762987.524201:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762987.524203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762987.524207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762987.524235:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762987.524242:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762987/real 1789762987] req@ffff9811c92c5180 x1876700048937856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762987.524254:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048937856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762987.524262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762987.524266:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762987.524270:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048937856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762987.779578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762988.419754:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762988.803557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762989.443755:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762989.827564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762990.467597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762990.851522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762991.491722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762991.875568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762992.515688:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762992.643587:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762992.643599:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762992.643605:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762992.643618:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2442 00000100:00080000:0.0:1789762992.643647:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762992.643657:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789762992.643701:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048937984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762992.643739:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762992.643771:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762992.643775:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762992.643784:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2442 00000100:00080000:0.0:1789762992.643793:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762992.643801:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762992.643818:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762992.643823:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048937984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789762992.643850:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048938112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762992.643874:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762992.643878:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762992.643881:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762992.643887:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2442 00000100:00080000:0.0:1789762992.643895:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762992.643901:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762992.643908:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762992.643929:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048938240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762992.643941:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762992.643955:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762992.643958:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762992.643966:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2442 00000100:00080000:0.0:1789762992.643973:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762992.643980:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789762992.643990:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789762992.643990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048938112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789762992.644021:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048938368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762992.644041:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762992.644045:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789762992.644048:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762992.644053:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2442 00000100:00080000:0.0:1789762992.644060:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762992.644070:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789762992.644072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938240:192.168.203.113@tcp:38:kworker.0 00000008:00100000:0.0:1789762992.644077:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789762992.644099:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048938496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762992.644153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762992.644195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762992.899566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762993.539682:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762993.923519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762994.563598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762994.947537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762995.587702:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762995.715816:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789762995.715869:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762995.715886:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762995.715928:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762995.715935:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762995.715941:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789762995.715947:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789762995.715992:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762992/real 1789762995] req@ffff9811c2e28e00 x1876700048938368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762995.716039:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048938368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762995.716049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762995.716053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762995.716057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762995.716065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762992/real 1789762995] req@ffff9811c2e2ad80 x1876700048938496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762995.716078:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048938496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762995.716085:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762995.716089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762995.716093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762995.716100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762992/real 1789762995] req@ffff9811c2e28700 x1876700048938112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762995.716114:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048938112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762995.716123:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762995.716133:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762995.716137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048938112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762995.716144:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762992/real 1789762995] req@ffff9811c2e29880 x1876700048938240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762995.716166:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048938240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762995.716175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762995.716178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762995.716182:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762995.716188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762992/real 1789762995] req@ffff9811c2e29c00 x1876700048937984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762995.716200:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048937984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762995.716207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762995.716211:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762995.716215:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048937984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789762995.971539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762996.611693:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762996.995520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762997.635757:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762997.763645:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762997.763656:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762997.763661:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762997.763680:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2447 00000100:00080000:1.0:1789762997.763691:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762997.763699:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789762997.763773:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048938624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762997.763808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762997.763811:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762997.763812:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762997.763816:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2447 00000100:00080000:1.0:1789762997.763827:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762997.763830:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762997.763848:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762997.763852:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048938624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762997.763867:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048938752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762997.763876:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762997.763879:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762997.763880:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762997.763893:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2447 00000100:00080000:1.0:1789762997.763896:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762997.763899:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762997.763903:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762997.763934:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048938880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762997.763943:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789762997.763954:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762997.763957:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762997.763958:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762997.763962:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2447 00000100:00080000:1.0:1789762997.763966:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762997.763979:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762997.763984:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789762997.763994:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048939008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762997.764001:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762997.764004:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789762997.764008:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762997.764010:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762997/real 1789762997] req@ffff9811c92c7b80 x1876700048938624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762997.764027:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2447 00000100:00080000:1.0:1789762997.764033:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762997.764039:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048938624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762997.764040:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789762997.764049:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789762997.764061:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762997.764066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762997.764071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048938624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789762997.764085:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048939136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762997.764119:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048938752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762997.764151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762997.764166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762997/real 1789762997] req@ffff9811c92c5880 x1876700048938752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.764179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048938752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.764187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762997.764191:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762997.764194:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048938752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762997.764200:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789762997.764220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762997.764232:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762997/real 1789762997] req@ffff9811c92c6d80 x1876700048938880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.764245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048938880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.764254:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762997.764258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762997.764261:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048938880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789762997.764268:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762997.764290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762997.764303:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762997/real 1789762997] req@ffff9811c92c5180 x1876700048939008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.764325:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048939008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.764334:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762997.764337:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762997.764340:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789762997.764345:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789762997.764366:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789762997.764378:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762997/real 1789762997] req@ffff9811c92c5500 x1876700048939136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.764390:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048939136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.764398:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762997.764402:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789762997.764405:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789762998.019539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789762998.659657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762999.043509:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789762999.683630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763000.067515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763000.707572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763001.091498:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763001.731576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763002.115508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763002.755544:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763002.883558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.883566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763002.883570:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763002.883578:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2452 00000100:00080000:2.0:1789763002.883594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763002.883601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763002.883633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048939264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763002.883681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.883686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763002.883688:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763002.883693:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2452 00000100:00080000:2.0:1789763002.883697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763002.883702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763002.883711:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763002.883712:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048939264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763002.883724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048939392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763002.883734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.883737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763002.883739:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763002.883743:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2452 00000100:00080000:2.0:1789763002.883759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763002.883763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763002.883768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763002.883771:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763002.883777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048939520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763002.883786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.883790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763002.883791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763002.883795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2452 00000100:00080000:2.0:1789763002.883805:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763002.883809:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763002.883809:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763002/real 1789763002] req@ffff9811d8639880 x1876700048939264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763002.883815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763002.883827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048939648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763002.883836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.883840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789763002.883841:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048939264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763002.883842:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763002.883846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2452 00000100:00080000:2.0:1789763002.883850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763002.883852:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763002.883853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763002.883856:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:2.0:1789763002.883858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763002.883860:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048939264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763002.883866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048939776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763002.883883:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048939392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763002.883909:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763002.883922:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763002/real 1789763002] req@ffff9811d8639500 x1876700048939392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763002.883931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048939392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763002.883938:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763002.883940:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763002.883942:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048939392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763002.883946:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763002.883960:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763002.883966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763002/real 1789763002] req@ffff9811d8639f80 x1876700048939520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763002.883974:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048939520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763002.883989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763002.883991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763002.883994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763002.883998:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763002.884012:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763002.884018:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763002/real 1789763002] req@ffff9811d863a680 x1876700048939648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763002.884026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048939648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763002.884031:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763002.884032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763002.884034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763002.884037:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763002.884047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763002.884052:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763002/real 1789763002] req@ffff9811d8638a80 x1876700048939776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763002.884059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048939776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763002.884064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763002.884066:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763002.884072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048939776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763003.139515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763003.779587:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763004.163504:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763004.803705:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763005.187550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763005.827638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763006.211569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763006.851655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763007.235512:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763007.875609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763008.003554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763008.003568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763008.003573:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763008.003586:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2457 00000100:00080000:2.0:1789763008.003600:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763008.003610:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763008.003674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048939904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763008.003706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763008.003730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763008.003733:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763008.003738:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2457 00000100:00080000:2.0:1789763008.003743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763008.003749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763008.003762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763008.003766:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048939904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763008.003780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048940032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763008.003795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763008.003799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763008.003802:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763008.003806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2457 00000100:00080000:2.0:1789763008.003826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763008.003832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763008.003839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763008.003841:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763008.003857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048940160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763008.003868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763008.003872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763008.003874:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763008.003879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2457 00000100:00080000:2.0:1789763008.003884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763008.003888:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763008/real 1789763008] req@ffff9811d863b800 x1876700048939904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763008.003892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763008.003906:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048939904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763008.003908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763008.003921:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763008.003924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048940288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763008.003928:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763008.003935:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048939904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763008.003939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763008.003943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763008.003945:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763008.003948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2457 00000100:00080000:2.0:1789763008.003952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763008.003955:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763008.003958:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763008.003963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048940032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763008.003969:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048940416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763008.003988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763008.004006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763008/real 1789763008] req@ffff9811d863a300 x1876700048940032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763008.004020:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048940032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763008.004038:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763008.004042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763008.004046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048940032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763008.004051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763008.004068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763008.004076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763008/real 1789763008] req@ffff9811d863b480 x1876700048940160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763008.004084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048940160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763008.004089:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763008.004092:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763008.004094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763008.004097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763008.004108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763008.004114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763008/real 1789763008] req@ffff9811d863aa00 x1876700048940288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763008.004121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048940288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763008.004126:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763008.004128:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763008.004135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763008.004148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763008.004161:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763008.004167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763008/real 1789763008] req@ffff9811d8638000 x1876700048940416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763008.004175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048940416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763008.004180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763008.004182:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763008.004183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763008.259535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763008.899612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763009.283515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763009.923637:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763010.307587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763010.947570:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763011.331525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763011.971544:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763012.355546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763012.995573:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763013.123533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763013.123543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763013.123547:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763013.123558:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2462 00000100:00080000:2.0:1789763013.123570:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763013.123579:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763013.123633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048940544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763013.123663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763013.123667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763013.123668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763013.123673:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2462 00000100:00080000:2.0:1789763013.123691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763013.123695:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763013.123705:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763013.123712:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048940544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763013.123720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048940672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763013.123731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763013.123734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763013.123735:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763013.123739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2462 00000100:00080000:2.0:1789763013.123742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763013.123746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763013.123750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763013.123758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048940800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763013.123764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763013.123767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763013.123769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763013.123772:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2462 00000100:00080000:2.0:1789763013.123775:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763013.123777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763013.123780:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763013.123788:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763013.123791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048940928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763013.123807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763013.123809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763013.123811:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763013.123813:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2462 00000100:00080000:2.0:1789763013.123816:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763013.123819:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763013.123823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763013.123835:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763013/real 1789763013] req@ffff9811d8638380 x1876700048940544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763013.123836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048941056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763013.123873:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048940544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763013.123895:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763013.123899:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763013.123903:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048940544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763013.123925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048940672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763013.123945:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763013.123960:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763013/real 1789763013] req@ffff9811d8638e00 x1876700048940672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763013.123975:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048940672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763013.123984:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763013.123988:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763013.123992:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048940672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763013.124000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763013.124025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763013.124040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763013/real 1789763013] req@ffff9811d8638700 x1876700048940800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763013.124054:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048940800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763013.124062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763013.124065:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763013.124067:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763013.124080:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763013.124100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763013.124111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763013/real 1789763013] req@ffff9811d8638000 x1876700048940928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763013.124121:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048940928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763013.124128:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763013.124132:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763013.124135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048940928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763013.124143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763013.124157:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763013.124163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763013/real 1789763013] req@ffff9811d863aa00 x1876700048941056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763013.124170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048941056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763013.124175:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763013.124177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763013.124179:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763013.379515:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763014.019580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763014.403557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763015.043632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763015.427521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763016.067735:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763016.451632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763017.091721:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763017.475634:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763018.115695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763018.243679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763018.243700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763018.243710:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763018.243758:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2467 00000100:00080000:0.0:1789763018.243775:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763018.243790:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763018.243863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048941184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763018.243922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763018.243928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763018.243931:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763018.243947:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2467 00000100:00080000:0.0:1789763018.243956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763018.243966:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763018.243987:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763018.244013:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048941184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763018.244055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048941312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763018.244096:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763018.244104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763018.244110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763018.244115:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763018.244126:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2467 00000100:00080000:0.0:1789763018.244146:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763018.244175:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763018.244191:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763018.244203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763018/real 1789763018] req@ffff9811c2e2a680 x1876700048941184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763018.244230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048941440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763018.244249:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048941184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763018.244283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763018.244298:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763018.244312:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763018.244322:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048941184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763018.244323:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763018.244335:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763018.244368:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2467 00000100:00080000:0.0:1789763018.244377:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763018.244385:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763018.244395:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763018.244405:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048941312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763018.244441:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048941568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763018.244551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763018.244580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763018.244584:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763018.244594:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2467 00000800:00000100:3.0:1789763018.244599:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763018.244606:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763018.244617:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763018.244627:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763018.244657:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763018/real 1789763018] req@ffff9811c2e29180 x1876700048941312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763018.244662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048941696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763018.244732:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048941312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763018.244755:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763018.244765:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763018.244774:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048941312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763018.244793:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763018.244854:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763018.244875:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763018/real 1789763018] req@ffff9811c2e2bb80 x1876700048941440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763018.244892:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048941440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763018.244930:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763018.244938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763018.244945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763018.244979:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763018.245049:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763018.245068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763018/real 1789763018] req@ffff9811c2e2b100 x1876700048941568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763018.245084:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048941568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763018.245094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763018.245099:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763018.245103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763018.245111:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763018.245135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763018.245147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763018/real 1789763018] req@ffff9811c2e2aa00 x1876700048941696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763018.245176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048941696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763018.245199:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763018.245205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763018.245226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048941696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763018.499643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763019.139704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763019.523635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763020.163763:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763020.547608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763021.187692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763021.571616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763022.211753:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763022.595615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763023.235704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763023.363701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763023.363724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763023.363730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763023.363743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2472 00000100:00080000:2.0:1789763023.363757:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763023.363767:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763023.363815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048941824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763023.363855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763023.363860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763023.363863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763023.363868:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2472 00000100:00080000:2.0:1789763023.363872:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763023.363876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763023.363920:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763023.363924:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048941824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763023.363945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048941952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763023.363965:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763023.363970:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763023.363992:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763023.363998:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2472 00000100:00080000:2.0:1789763023.364005:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763023.364012:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763023.364018:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763023.364030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763023.364038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048942080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763023.364054:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763023.364060:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763023.364062:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763023.364068:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2472 00000100:00080000:2.0:1789763023.364072:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763023.364077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763023.364083:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763023.364100:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048942208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763023.364113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763023/real 1789763023] req@ffff9811d863b480 x1876700048941824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763023.364121:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763023.364136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763023.364149:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763023.364162:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048941824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763023.364163:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2472 00000100:00080000:2.0:1789763023.364175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763023.364188:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763023.364200:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763023.364204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763023.364220:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763023.364222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048942336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763023.364240:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048941824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763023.364291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048941952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763023.364333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763023.364351:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763023/real 1789763023] req@ffff9811d863a300 x1876700048941952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763023.364367:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048941952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763023.364378:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763023.364394:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763023.364399:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048941952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763023.364407:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763023.364434:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763023.364447:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763023/real 1789763023] req@ffff9811d863b800 x1876700048942080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763023.364514:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048942080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763023.364553:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763023.364558:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763023.364563:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763023.364572:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763023.364625:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763023.364641:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763023/real 1789763023] req@ffff9811d8638a80 x1876700048942208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763023.364656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048942208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763023.364667:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763023.364671:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763023.364675:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763023.364682:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763023.364718:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763023.364731:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763023/real 1789763023] req@ffff9811d863a680 x1876700048942336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763023.364752:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048942336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763023.364762:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763023.364765:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763023.364769:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763023.619608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763024.259732:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763024.643617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763025.283727:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763025.667571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763026.307719:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763026.691592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763027.331769:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763027.715591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763028.355738:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763028.483651:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763028.483666:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763028.483672:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763028.483688:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2478 00000100:00080000:1.0:1789763028.483703:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763028.483714:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763028.483774:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048942464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763028.483822:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763028.483829:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763028.483833:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763028.483841:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2478 00000100:00080000:1.0:1789763028.483848:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763028.483856:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763028.483875:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763028.483887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048942464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763028.483934:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048942592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763028.483959:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763028.483964:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763028.483966:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763028.483972:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2478 00000100:00080000:1.0:1789763028.483980:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763028.483985:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763028.483993:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763028.483998:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763028.484018:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048942720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763028.484032:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763028.484036:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763028.484038:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763028.484043:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2478 00000100:00080000:1.0:1789763028.484046:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763028.484050:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763028.484057:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763028.484069:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048942848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763028.484079:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763028.484083:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763028.484086:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763028.484092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763028/real 1789763028] req@ffff9811c92c4000 x1876700048942464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763028.484121:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2478 00000100:00080000:1.0:1789763028.484131:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763028.484132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048942464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763028.484146:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763028.484159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789763028.484160:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763028.484174:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763028.484181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048942464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763028.484188:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048942976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763028.484224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048942592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763028.484270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763028.484287:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763028/real 1789763028] req@ffff9811c92c4e00 x1876700048942592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763028.484301:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048942592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763028.484310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763028.484314:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763028.484317:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048942592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763028.484323:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763028.484341:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763028.484351:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763028/real 1789763028] req@ffff9811c92c7480 x1876700048942720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763028.484363:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048942720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763028.484371:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763028.484374:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763028.484377:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763028.484383:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763028.484400:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763028.484409:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763028/real 1789763028] req@ffff9811c92c5c00 x1876700048942848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763028.484441:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048942848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763028.484450:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763028.484453:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763028.484456:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763028.484544:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763028.484583:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763028.484599:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763028/real 1789763028] req@ffff9811c92c6680 x1876700048942976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763028.484612:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048942976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763028.484622:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763028.484625:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763028.484628:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048942976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763028.739653:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763029.379753:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763029.763594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763030.403765:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763030.787593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763031.427733:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763031.811615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763032.451683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763032.835584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763033.475703:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763033.603646:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763033.603662:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763033.603669:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763033.603685:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2483 00000100:00080000:2.0:1789763033.603703:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763033.603715:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763033.603778:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048943104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763033.603816:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763033.603821:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763033.603824:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763033.603830:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2483 00000100:00080000:2.0:1789763033.603834:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763033.603839:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763033.603854:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763033.603881:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048943232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763033.603904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048943104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763033.603948:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763033.603971:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763033.603976:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763033.603985:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2483 00000100:00080000:2.0:1789763033.603995:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763033.604002:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763033.604012:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763033.604021:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763033.604043:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048943360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763033.604067:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763033.604074:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763033.604078:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763033.604087:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2483 00000100:00100000:3.0:1789763033.604095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763033/real 1789763033] req@ffff9811d8639f80 x1876700048943104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763033.604099:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763033.604111:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763033.604127:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763033.604160:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048943488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763033.604173:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048943104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763033.604180:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763033.604194:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763033.604201:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763033.604218:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2483 00000100:00080000:3.0:1789763033.604223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763033.604234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763033.604245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048943104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763033.604247:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763033.604265:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763033.604278:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763033.604297:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048943232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763033.604309:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048943616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763033.604348:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763033.604375:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763033/real 1789763033] req@ffff9811d8639500 x1876700048943232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763033.604393:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048943232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763033.604404:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763033.604409:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763033.604413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048943232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763033.604419:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763033.604507:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763033.604537:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763033/real 1789763033] req@ffff9811d8639880 x1876700048943360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763033.604565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048943360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763033.604606:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763033.604611:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763033.604615:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763033.604624:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763033.604659:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763033.604674:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763033/real 1789763033] req@ffff9811d863b100 x1876700048943488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763033.604688:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048943488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763033.604698:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763033.604701:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763033.604705:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763033.604726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763033.604747:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763033.604759:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763033/real 1789763033] req@ffff9811d8639c00 x1876700048943616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763033.604773:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048943616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763033.604783:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763033.604786:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763033.604800:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048943616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763033.859612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763034.499697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763034.883528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763035.523673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763035.907527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763036.547614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763036.931536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763037.571580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763037.955591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763038.595736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763038.723619:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763038.723632:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763038.723638:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763038.723651:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2488 00000100:00080000:1.0:1789763038.723666:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763038.723676:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763038.723725:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048943744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763038.723764:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763038.723769:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763038.723791:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763038.723799:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2488 00000100:00080000:1.0:1789763038.723805:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763038.723811:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763038.723824:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763038.723842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048943744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763038.723847:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048943872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763038.723874:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763038.723879:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763038.723881:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763038.723888:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2488 00000100:00080000:1.0:1789763038.723893:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763038.723897:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763038.723904:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763038.723919:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048944000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763038.723933:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763038.723939:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763038.723941:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763038.723945:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2488 00000100:00080000:1.0:1789763038.723951:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763038.723960:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763038.723965:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763038.723975:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763038.724003:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048944128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763038.724015:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763038.724020:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763038.724023:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763038.724027:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2488 00000100:00080000:1.0:1789763038.724034:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763038.724039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763038/real 1789763038] req@ffff9811c92c4380 x1876700048943744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763038.724045:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763038.724057:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763038.724090:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048944256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763038.724094:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048943744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763038.724135:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763038.724142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763038.724150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048943744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763038.724193:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048943872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763038.724261:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763038.724288:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763038/real 1789763038] req@ffff9811c92c4a80 x1876700048943872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763038.724316:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048943872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763038.724329:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763038.724333:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763038.724351:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048943872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763038.724360:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763038.724394:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763038.724441:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763038/real 1789763038] req@ffff9811c92c5f80 x1876700048944000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763038.724460:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048944000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763038.724521:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763038.724529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763038.724537:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763038.724551:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763038.724596:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763038.724614:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763038/real 1789763038] req@ffff9811c92c6a00 x1876700048944128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763038.724631:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048944128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763038.724641:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763038.724645:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763038.724649:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763038.724657:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763038.724679:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763038.724705:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763038/real 1789763038] req@ffff9811c92c6300 x1876700048944256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763038.724720:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048944256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763038.724731:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763038.724735:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763038.724739:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763038.979602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763039.619694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763040.003565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763040.643657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763041.027573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763041.667671:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763042.051535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763042.691708:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763043.075623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763043.715687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763043.843566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763043.843574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763043.843578:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763043.843586:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2493 00000100:00080000:0.0:1789763043.843613:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763043.843621:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763043.843660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048944384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763043.843689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763043.843694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763043.843696:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763043.843700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2493 00000100:00080000:0.0:1789763043.843704:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763043.843708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763043.843718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763043.843734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048944512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763043.843748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763043.843754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763043.843757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763043.843769:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048944384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763043.843770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2493 00000100:00080000:0.0:1789763043.843783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763043.843792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763043.843799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763043.843820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048944640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763043.843835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763043.843844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763043.843846:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763043.843850:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2493 00000100:00080000:0.0:1789763043.843856:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763043.843860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763043.843865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763043.843876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048944768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763043.843884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763043.843888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763043.843891:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763043.843902:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2493 00000800:00000100:3.0:1789763043.843902:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763043.843915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763043.843919:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763043.843923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763043.843932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048944896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763043.844004:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763043/real 1789763043] req@ffff9811c2e28380 x1876700048944384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.844034:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048944384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.844062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.844068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763043.844073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048944384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763043.844111:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048944512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763043.844147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763043.844187:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763043/real 1789763043] req@ffff9811c2e28a80 x1876700048944512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.844213:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048944512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.844225:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.844229:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763043.844233:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048944512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.844241:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763043.844265:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763043.844277:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763043/real 1789763043] req@ffff9811c2e29f80 x1876700048944640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.844291:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048944640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.844301:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.844304:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763043.844308:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.844314:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763043.844334:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763043.844344:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763043/real 1789763043] req@ffff9811c2e2b800 x1876700048944768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.844358:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048944768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.844410:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.844415:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763043.844419:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763043.844428:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763043.844458:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763043.844514:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763043/real 1789763043] req@ffff9811c2e2b480 x1876700048944896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.844529:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048944896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.844539:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.844543:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763043.844547:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048944896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763044.099582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763044.739721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763045.123532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763045.763582:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763046.147539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763046.787602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763047.171533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763047.811703:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763048.195580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763048.835651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763048.963625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763048.963644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763048.963651:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763048.963670:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2498 00000100:00080000:0.0:1789763048.963688:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763048.963703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763048.963768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048945024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763048.963853:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763048.963859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763048.963863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763048.963872:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2498 00000100:00080000:0.0:1789763048.963879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763048.963900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763048.963924:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763048.963957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048945152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763048.963990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763048.964000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763048.964005:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763048.964017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048945024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763048.964019:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2498 00000100:00080000:0.0:1789763048.964034:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763048.964064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763048.964077:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763048.964109:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048945280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763048.964129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763048.964134:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763048.964141:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763048.964142:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763048.964152:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2498 00000100:00080000:0.0:1789763048.964176:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763048.964187:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763048.964198:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763048.964222:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048945408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763048.964237:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763048.964242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763048.964249:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763048.964251:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763048/real 1789763048] req@ffff9811c2e2a300 x1876700048945024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763048.964258:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2498 00000100:00080000:0.0:1789763048.964270:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763048.964283:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763048.964298:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763048.964315:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048945536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763048.964323:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048945024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763048.964388:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763048.964397:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763048.964405:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048945024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763048.964418:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048945152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763048.964534:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763048.964585:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763048/real 1789763048] req@ffff9811c2e28000 x1876700048945152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763048.964609:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048945152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763048.964635:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763048.964642:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763048.964656:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048945152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763048.964716:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763048.964745:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763048.964760:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763048/real 1789763048] req@ffff9811c2e29500 x1876700048945280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763048.964774:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048945280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763048.964784:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763048.964787:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763048.964791:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763048.964810:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763048.964836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763048.964849:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763048/real 1789763048] req@ffff9811c2e2b480 x1876700048945408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763048.964863:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048945408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763048.964873:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763048.964876:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763048.964880:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763048.964886:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763048.964904:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763048.964915:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763048/real 1789763048] req@ffff9811c2e2b800 x1876700048945536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763048.964929:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048945536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763048.964937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763048.964941:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763048.964944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763049.219604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763049.859673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763050.243615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763050.883686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763051.267635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763051.907691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763052.291613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763052.931701:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763053.315602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763053.955651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763054.083621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763054.083644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763054.083652:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763054.083673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2503 00000100:00080000:2.0:1789763054.083694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763054.083706:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763054.083770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048945664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763054.083814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763054.083819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763054.083822:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763054.083827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2503 00000100:00080000:2.0:1789763054.083832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763054.083838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763054.083852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763054.083870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048945792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763054.083903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048945664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763054.083906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763054.083954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763054.083964:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763054.083974:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2503 00000100:00080000:2.0:1789763054.083988:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763054.083995:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763054.084005:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763054.084021:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763054.084031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048945920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763054.084053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763054.084058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763054.084061:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763054.084067:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2503 00000100:00080000:2.0:1789763054.084073:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763054.084077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763054.084083:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763054.084106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048946048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763054.084122:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763054/real 1789763054] req@ffff9811d863ad80 x1876700048945664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763054.084133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763054.084164:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763054.084173:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763054.084204:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2503 00000100:00080000:2.0:1789763054.084216:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763054.084226:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763054.084228:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048945664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763054.084258:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789763054.084263:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763054.084273:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763054.084283:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048946176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763054.084315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048945664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763054.084386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048945792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763054.084440:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763054.084501:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763054/real 1789763054] req@ffff9811d863bb80 x1876700048945792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763054.084521:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048945792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763054.084534:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763054.084538:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763054.084543:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048945792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763054.084552:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763054.084580:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763054.084594:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763054/real 1789763054] req@ffff9811d8639180 x1876700048945920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763054.084627:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048945920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763054.084638:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763054.084642:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763054.084646:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048945920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763054.084654:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763054.084697:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763054.084711:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763054/real 1789763054] req@ffff9811d8639c00 x1876700048946048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763054.084726:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048946048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763054.084736:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763054.084740:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763054.084744:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763054.084751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763054.084771:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763054.084781:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763054/real 1789763054] req@ffff9811d863b100 x1876700048946176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763054.084794:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048946176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763054.084821:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763054.084827:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763054.084831:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763054.339572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763054.979745:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763055.363650:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763056.003727:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763056.387531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763057.027592:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763057.411580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763058.051643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763058.435633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763059.075719:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763059.203660:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763059.203672:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763059.203678:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763059.203691:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2508 00000100:00080000:3.0:1789763059.203706:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763059.203715:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763059.203762:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048946304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763059.203804:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763059.203809:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763059.203812:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763059.203817:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2508 00000100:00080000:3.0:1789763059.203821:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763059.203825:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763059.203837:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763059.203853:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048946432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763059.203869:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763059.203874:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763059.203877:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763059.203894:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048946304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763059.203905:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2508 00000100:00080000:3.0:1789763059.203913:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763059.203919:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763059.203927:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763059.203944:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048946560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763059.203960:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763059.203965:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763059.203968:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763059.203974:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2508 00000100:00080000:3.0:1789763059.203979:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763059.203983:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763059.203990:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763059.204004:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048946688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763059.204015:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763059.204020:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763059.204023:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763059.204027:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2508 00000100:00080000:3.0:1789763059.204031:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763059.204036:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763059.204042:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763059.204055:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048946816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763059.204080:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048946432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763059.204156:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763059.204184:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763059.204234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763059.459563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763060.099717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763060.483590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763061.123698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763061.507574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763062.147587:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763062.275729:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789763062.275782:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763062.275799:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763062.275833:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763062.275844:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763062.275855:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763062.275871:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789763062.275876:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763059/real 1789763062] req@ffff9811e1a22680 x1876700048946560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763062.275904:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048946560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763062.275916:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763062.275921:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763062.275926:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763062.275935:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763059/real 1789763062] req@ffff9811e1a21c00 x1876700048946688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763062.275967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048946688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763062.275976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763062.275980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763062.275983:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763062.275990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763059/real 1789763062] req@ffff9811e1a21880 x1876700048946816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763062.276001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048946816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763062.276009:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763062.276013:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763062.276023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048946816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763062.276031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763059/real 1789763062] req@ffff9811e1a22300 x1876700048946304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763062.276044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048946304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763062.276052:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763062.276056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763062.276060:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048946304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763062.276067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763059/real 1789763062] req@ffff9811e1a21180 x1876700048946432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763062.276088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048946432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763062.276097:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763062.276100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763062.276103:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048946432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763062.531582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763063.171779:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763063.555591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763064.195691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763064.323655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763064.323668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763064.323673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763064.323686:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2513 00000100:00080000:0.0:1789763064.323700:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763064.323709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763064.323756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048946944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763064.323815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763064.323823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763064.323827:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763064.323836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2513 00000100:00080000:0.0:1789763064.323845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763064.323852:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763064.323868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763064.323894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048947072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763064.323904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048946944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763064.323917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763064.323924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763064.323927:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763064.323933:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2513 00000100:00080000:0.0:1789763064.323939:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763064.323945:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763064.323954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763064.323970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048947200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763064.323987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763064.323994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1789763064.323994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763064.323998:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763064.324004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2513 00000100:00080000:0.0:1789763064.324023:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763064.324033:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763064.324043:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763064.324055:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763064/real 1789763064] req@ffff9811c2e29f80 x1876700048946944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763064.324062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048947328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763064.324081:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763064.324088:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789763064.324088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048946944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763064.324092:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763064.324098:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2513 00000100:00080000:0.0:1789763064.324104:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763064.324108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763064.324110:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789763064.324115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:0.0:1789763064.324118:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763064.324124:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048946944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763064.324134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048947456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763064.324160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048947072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763064.324205:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763064.324268:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763064/real 1789763064] req@ffff9811c2e28a80 x1876700048947072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763064.324295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048947072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763064.324338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763064.324346:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763064.324354:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048947072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763064.324367:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763064.324416:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763064.324441:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763064/real 1789763064] req@ffff9811c2e28380 x1876700048947200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763064.324528:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048947200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763064.324547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763064.324554:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763064.324560:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763064.324573:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763064.324627:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763064.324652:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763064/real 1789763064] req@ffff9811c2e2aa00 x1876700048947328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763064.324677:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048947328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763064.324693:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763064.324699:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763064.324724:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763064.324738:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763064.324784:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763064.324808:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763064/real 1789763064] req@ffff9811c2e2b100 x1876700048947456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763064.324831:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048947456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763064.324847:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763064.324854:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763064.324860:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763064.579566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763065.219674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763065.603554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763066.243689:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763066.627539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763067.267726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763067.651518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763068.291699:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763068.675547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763069.315690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763069.443565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763069.443577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763069.443581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763069.443599:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2519 00000100:00080000:0.0:1789763069.443611:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763069.443619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763069.443664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048947584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763069.443738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763069.443743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763069.443746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763069.443752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2519 00000100:00080000:0.0:1789763069.443757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763069.443762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763069.443774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763069.443792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048947712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763069.443802:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048947584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763069.443807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763069.443811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763069.443814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763069.443818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2519 00000100:00080000:0.0:1789763069.443823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763069.443827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763069.443832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763069.443843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048947840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763069.443854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763069.443858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763069.443861:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763069.443865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2519 00000800:00000100:1.0:1789763069.443870:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763069.443877:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763069.443882:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763069.443888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763069.443902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048947968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763069.443915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763069.443916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763069/real 1789763069] req@ffff9811c2e2bb80 x1876700048947584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763069.443919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763069.443922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763069.443927:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2519 00000100:00080000:0.0:1789763069.443932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763069.443936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763069.443938:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048947584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763069.443943:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789763069.443952:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763069.443954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048948096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763069.443957:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763069.443962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048947584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763069.444004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048947712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763069.444035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763069.444054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763069/real 1789763069] req@ffff9811c2e29180 x1876700048947712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763069.444072:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048947712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763069.444086:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763069.444090:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763069.444094:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048947712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763069.444102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763069.444124:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763069.444134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763069/real 1789763069] req@ffff9811c2e2a680 x1876700048947840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763069.444144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048947840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763069.444152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763069.444154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763069.444157:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763069.444161:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763069.444175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763069.444208:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763069/real 1789763069] req@ffff9811c2e29c00 x1876700048947968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763069.444254:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048947968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763069.444265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763069.444269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763069.444274:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048947968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763069.444282:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763069.444310:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763069.444322:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763069/real 1789763069] req@ffff9811c2e29880 x1876700048948096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763069.444332:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048948096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763069.444340:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763069.444342:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763069.444345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763069.699510:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763070.339757:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763070.723536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763071.363762:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763071.747586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763072.387712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763072.771553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763073.411628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763073.795589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763074.435700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763074.563638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763074.563651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763074.563656:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763074.563670:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2524 00000100:00080000:2.0:1789763074.563685:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763074.563694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763074.563743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048948224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763074.563782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763074.563787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763074.563790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763074.563795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2524 00000100:00080000:2.0:1789763074.563799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763074.563803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763074.563839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763074.563862:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048948224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763074.563874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048948352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763074.563904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763074.563913:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763074.563918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763074.563926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2524 00000100:00080000:2.0:1789763074.563934:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763074.563941:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763074.563950:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763074.563952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763074.563977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048948480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763074.563993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763074.563998:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763074.564000:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763074.564004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2524 00000100:00080000:2.0:1789763074.564009:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763074.564015:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763074.564022:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763074.564027:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763074/real 1789763074] req@ffff9811d8639880 x1876700048948224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763074.564046:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048948608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763074.564062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763074.564069:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048948224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763074.564071:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763074.564098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763074.564109:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763074.564112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763074.564125:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2524 00000100:00100000:1.0:1789763074.564127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048948224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763074.564137:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763074.564175:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763074.564187:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763074.564196:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048948352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763074.564218:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048948736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763074.564240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763074.564267:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763074/real 1789763074] req@ffff9811d8639500 x1876700048948352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763074.564295:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048948352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763074.564313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763074.564320:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763074.564327:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048948352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763074.564341:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763074.564381:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763074.564404:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763074/real 1789763074] req@ffff9811d8639f80 x1876700048948480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763074.564425:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048948480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763074.564436:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763074.564440:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763074.564444:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763074.564540:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763074.564571:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763074.564586:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763074/real 1789763074] req@ffff9811d863a680 x1876700048948608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763074.564600:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048948608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763074.564611:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763074.564614:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763074.564618:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763074.564656:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763074.564679:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763074.564690:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763074/real 1789763074] req@ffff9811d8638a80 x1876700048948736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763074.564705:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048948736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763074.564714:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763074.564718:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763074.564721:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048948736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763074.819563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763075.459619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763075.843544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763076.483611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763076.867564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763077.507670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763077.891584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763078.531678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763078.915551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763079.555665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763079.683640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763079.683653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763079.683659:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763079.683673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2529 00000100:00080000:2.0:1789763079.683687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763079.683697:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763079.683748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048948864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763079.683798:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763079.683806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763079.683810:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763079.683820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2529 00000100:00080000:2.0:1789763079.683826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763079.683834:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763079.683850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763079.683873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048948992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763079.683874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048948864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763079.683922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763079.683928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763079.683931:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763079.683950:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2529 00000100:00080000:2.0:1789763079.683957:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763079.683964:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763079.683973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763079.683992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048949120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763079.683995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763079.684010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763079.684016:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763079.684019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763079.684025:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2529 00000100:00080000:2.0:1789763079.684031:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763079.684037:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763079.684054:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763079.684067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763079/real 1789763079] req@ffff9811d863b800 x1876700048948864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763079.684070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048949248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763079.684091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763079.684097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763079.684101:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763079.684126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048948864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763079.684139:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2529 00000100:00080000:1.0:1789763079.684148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763079.684155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763079.684158:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763079.684163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048948864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763079.684166:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763079.684186:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763079.684206:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048949376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763079.684212:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048948992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763079.684255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763079.684281:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763079/real 1789763079] req@ffff9811d863a300 x1876700048948992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763079.684307:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048948992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763079.684322:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763079.684326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763079.684330:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048948992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763079.684361:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763079.684392:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763079.684407:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763079/real 1789763079] req@ffff9811d863b480 x1876700048949120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763079.684441:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048949120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763079.684451:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763079.684455:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763079.684502:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763079.684511:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763079.684541:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763079.684557:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763079/real 1789763079] req@ffff9811d863aa00 x1876700048949248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763079.684572:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048949248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763079.684582:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763079.684585:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763079.684590:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763079.684622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763079.684642:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763079.684654:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763079/real 1789763079] req@ffff9811d8638000 x1876700048949376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763079.684667:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048949376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763079.684699:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763079.684707:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763079.684713:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763079.939614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763080.579661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763080.963570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763081.603732:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763081.987539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763082.627612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763083.011599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763083.651698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763084.035646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763084.675718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763084.803641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763084.803656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763084.803662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763084.803689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2534 00000100:00080000:0.0:1789763084.803703:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763084.803714:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763084.803762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048949504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763084.803795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763084.803812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763084.803816:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763084.803823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2534 00000100:00080000:0.0:1789763084.803831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763084.803842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763084.803860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763084.803889:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048949504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763084.803893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048949632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763084.803930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763084.803947:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763084.803950:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763084.803976:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2534 00000100:00080000:0.0:1789763084.804008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763084.804023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789763084.804024:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789763084.804044:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763084.804101:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048949760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763084.804126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763084/real 1789763084] req@ffff9811c2e28700 x1876700048949504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763084.804145:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763084.804169:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789763084.804176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048949504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763084.804181:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763084.804200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2534 00000100:00080000:0.0:1789763084.804216:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763084.804226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763084.804235:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789763084.804239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763084.804246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048949504:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763084.804269:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763084.804290:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048949632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763084.804346:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048949888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763084.804369:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763084.804371:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763084.804378:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763084.804381:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763084.804390:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2534 00000100:00080000:0.0:1789763084.804398:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763084.804407:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763084.804408:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763084/real 1789763084] req@ffff9811c2e2ad80 x1876700048949632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763084.804419:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763084.804434:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048949632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763084.804446:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048950016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763084.804456:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763084.804578:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763084.804591:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048949632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763084.804609:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763084.804660:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763084.804710:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763084/real 1789763084] req@ffff9811c2e28e00 x1876700048949760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763084.804727:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048949760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763084.804738:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763084.804743:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763084.804747:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763084.804780:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763084.804809:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763084.804838:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763084/real 1789763084] req@ffff9811c2e28700 x1876700048949888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763084.804848:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048949888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763084.804855:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763084.804860:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763084.804863:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048949888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763084.804872:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763084.804907:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763084.804919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763084/real 1789763084] req@ffff9811c2e29880 x1876700048950016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763084.804928:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048950016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763084.804933:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763084.804935:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763084.804937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763085.059577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763085.699722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763086.083613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763086.723732:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763087.107550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763087.747718:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763088.131595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763088.771723:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763089.155606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763089.795770:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763089.923551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763089.923568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763089.923574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763089.923587:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2539 00000100:00080000:0.0:1789763089.923600:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763089.923608:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763089.923656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048950144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763089.923693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763089.923699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763089.923718:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763089.923723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2539 00000100:00080000:0.0:1789763089.923728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763089.923732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763089.923744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763089.923750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048950144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763089.923760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048950272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763089.923776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763089.923780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763089.923782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763089.923787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2539 00000100:00080000:0.0:1789763089.923792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763089.923798:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763089.923804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763089.923815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048950400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763089.923822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763089.923825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763089.923827:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763089.923830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2539 00000100:00080000:0.0:1789763089.923833:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763089.923836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763089.923840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789763089.923842:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763089.923856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048950528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763089.923863:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763089.923866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763089.923868:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763089.923870:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2539 00000100:00080000:0.0:1789763089.923873:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763089.923875:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763089.923879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763089.923888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048950656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763089.923897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763089/real 1789763089] req@ffff9811c2e29c00 x1876700048950144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763089.923917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048950144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763089.923928:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763089.923931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763089.923935:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048950144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763089.923960:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048950272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763089.923978:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763089.923987:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763089/real 1789763089] req@ffff9811c2e2a680 x1876700048950272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763089.924009:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048950272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763089.924016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763089.924018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763089.924020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048950272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763089.924025:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763089.924059:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763089.924067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763089/real 1789763089] req@ffff9811c2e29180 x1876700048950400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763089.924076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048950400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763089.924082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763089.924084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763089.924086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763089.924090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763089.924103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763089.924109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763089/real 1789763089] req@ffff9811c2e2bb80 x1876700048950528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763089.924117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048950528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763089.924132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763089.924135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763089.924137:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763089.924141:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763089.924155:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763089.924161:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763089/real 1789763089] req@ffff9811c2e2b100 x1876700048950656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763089.924169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048950656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763089.924174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763089.924176:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763089.924177:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048950656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763090.179589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763090.819809:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763091.203582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763091.843723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763092.227594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763092.867716:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763093.251589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763093.891729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763094.275567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763094.915690:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763095.043611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763095.043627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763095.043634:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763095.043648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2544 00000100:00080000:0.0:1789763095.043662:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763095.043672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763095.043719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048950784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763095.043758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763095.043763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763095.043782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763095.043789:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2544 00000100:00080000:0.0:1789763095.043795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763095.043801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763095.043816:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763095.043836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048950912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763095.043844:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048950784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763095.043869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763095.043876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763095.043878:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763095.043885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2544 00000100:00080000:0.0:1789763095.043890:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763095.043895:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763095.043901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763095.043918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048951040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763095.043931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763095.043935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763095.043938:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763095.043946:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763095.043948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2544 00000100:00080000:0.0:1789763095.043959:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763095.043964:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763095.043971:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763095.044028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048951168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763095.044044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763095/real 1789763095] req@ffff9811c2e2aa00 x1876700048950784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763095.044046:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763095.044059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763095.044065:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763095.044080:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2544 00000100:00080000:0.0:1789763095.044093:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763095.044101:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763095.044109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763095.044129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048951296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763095.044129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048950784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763095.044187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763095.044208:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763095.044215:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048950784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763095.044249:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048950912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763095.044284:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763095.044302:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763095/real 1789763095] req@ffff9811c2e28380 x1876700048950912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763095.044319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048950912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763095.044329:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763095.044334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763095.044338:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048950912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763095.044345:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763095.044365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763095.044377:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763095/real 1789763095] req@ffff9811c2e28a80 x1876700048951040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763095.044390:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048951040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763095.044400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763095.044404:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763095.044408:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763095.044414:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763095.044449:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763095.044526:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763095/real 1789763095] req@ffff9811c2e29f80 x1876700048951168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763095.044555:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048951168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763095.044570:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763095.044574:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763095.044579:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763095.044587:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763095.044619:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763095.044634:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763095/real 1789763095] req@ffff9811c2e2b800 x1876700048951296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763095.044648:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048951296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763095.044657:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763095.044661:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763095.044665:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763095.299567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763095.939700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763096.323576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763096.963737:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763097.347564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763097.987714:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763098.371590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763099.011739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763099.395581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763100.035753:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763100.163654:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763100.163673:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763100.163705:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763100.163719:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2549 00000100:00080000:1.0:1789763100.163733:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763100.163742:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763100.163804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048951424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763100.163836:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763100.163843:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763100.163845:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763100.163850:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2549 00000100:00080000:1.0:1789763100.163853:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763100.163857:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763100.163870:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763100.163882:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048951552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763100.163895:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763100.163896:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048951424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763100.163904:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763100.163908:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763100.163912:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2549 00000100:00080000:1.0:1789763100.163915:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763100.163918:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763100.163922:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763100.163935:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048951680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763100.163976:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763100.163980:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763100.163982:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763100.163992:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2549 00000100:00080000:1.0:1789763100.163998:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763100.164002:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763100.164008:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763100.164022:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048951808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763100.164030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763100.164039:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763100.164043:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763100.164045:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763100.164050:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2549 00000100:00080000:1.0:1789763100.164054:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763100.164057:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763100.164061:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763100.164070:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048951936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763100.164109:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763100/real 1789763100] req@ffff9811c92c4700 x1876700048951424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763100.164139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048951424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763100.164157:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763100.164165:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763100.164171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048951424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763100.164222:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048951552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763100.164270:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763100.164285:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763100/real 1789763100] req@ffff9811c92c7800 x1876700048951552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763100.164298:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048951552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763100.164306:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763100.164309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763100.164313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048951552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763100.164319:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763100.164336:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763100.164345:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763100/real 1789763100] req@ffff9811c92c7100 x1876700048951680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763100.164355:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048951680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763100.164361:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763100.164364:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763100.164366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763100.164371:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763100.164386:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763100.164393:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763100/real 1789763100] req@ffff9811c92c6300 x1876700048951808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763100.164420:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048951808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763100.164431:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763100.164436:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763100.164441:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763100.164449:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763100.164505:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763100.164517:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763100/real 1789763100] req@ffff9811c92c6a00 x1876700048951936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763100.164528:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048951936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763100.164536:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763100.164539:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763100.164542:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048951936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763100.419571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763101.059753:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763101.443611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763102.083710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763102.467592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763103.107604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763103.491558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763104.131726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763104.515602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763105.155699:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763105.283603:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763105.283633:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763105.283638:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763105.283652:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2554 00000100:00080000:3.0:1789763105.283666:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763105.283675:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763105.283719:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048952064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763105.283759:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763105.283764:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763105.283767:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763105.283772:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2554 00000100:00080000:3.0:1789763105.283776:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763105.283781:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763105.283794:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763105.283810:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048952192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763105.283825:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048952064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763105.283832:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763105.283840:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763105.283843:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763105.283849:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2554 00000100:00080000:3.0:1789763105.283853:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763105.283859:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763105.283865:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763105.283881:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048952320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763105.283905:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763105.283910:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763105.283912:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763105.283939:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2554 00000800:00000100:1.0:1789763105.283948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763105.283950:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763105.283960:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763105.283975:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763105.283995:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048952448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763105.284008:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763105.284013:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763105.284015:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763105.284020:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2554 00000100:00080000:3.0:1789763105.284024:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763105.284030:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763105.284036:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763105.284047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763105/real 1789763105] req@ffff9811e1a22d80 x1876700048952064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763105.284056:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048952576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763105.284110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048952064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763105.284129:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763105.284135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763105.284141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048952064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763105.284171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048952192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763105.284204:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763105.284220:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763105/real 1789763105] req@ffff9811e1a22a00 x1876700048952192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763105.284236:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048952192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763105.284247:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763105.284251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763105.284255:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048952192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763105.284262:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763105.284282:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763105.284293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763105/real 1789763105] req@ffff9811e1a21180 x1876700048952320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763105.284307:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048952320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763105.284336:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763105.284340:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763105.284344:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763105.284352:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763105.284394:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763105.284409:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763105/real 1789763105] req@ffff9811e1a22300 x1876700048952448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763105.284424:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048952448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763105.284434:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763105.284438:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763105.284442:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763105.284448:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763105.284505:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763105.284524:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763105/real 1789763105] req@ffff9811e1a21880 x1876700048952576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763105.284549:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048952576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763105.284565:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763105.284570:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763105.284594:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763105.539592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763106.179713:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763106.563553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763107.203585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763107.587571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763108.227733:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763108.611580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763109.251684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763109.635602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763110.275704:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763110.403600:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763110.403613:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763110.403618:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763110.403630:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2559 00000100:00080000:0.0:1789763110.403664:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763110.403676:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763110.403740:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048952704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763110.403781:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763110.403786:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763110.403789:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763110.403821:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2559 00000100:00080000:0.0:1789763110.403828:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763110.403835:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763110.403850:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763110.403870:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048952832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763110.403899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048952704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763110.403936:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763110.403939:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763110.403941:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763110.403945:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2559 00000100:00080000:0.0:1789763110.403949:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763110.403952:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763110.403957:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763110.403968:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048952960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763110.403976:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763110.403979:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763110.403981:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763110.403983:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2559 00000100:00080000:0.0:1789763110.403998:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763110.404000:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763110.404004:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789763110.404010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763110.404014:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048953088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763110.404029:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763110.404032:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763110.404034:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763110.404038:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2559 00000100:00080000:0.0:1789763110.404041:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763110.404044:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763110.404048:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763110.404059:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048953216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763110.404068:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763110/real 1789763110] req@ffff9811c2e2b480 x1876700048952704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763110.404089:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048952704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763110.404099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763110.404102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763110.404105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048952704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763110.404131:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048952832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763110.404149:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763110.404158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763110/real 1789763110] req@ffff9811c2e29500 x1876700048952832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763110.404167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048952832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763110.404177:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763110.404181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763110.404185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048952832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763110.404204:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763110.404230:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763110.404241:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763110/real 1789763110] req@ffff9811c2e28000 x1876700048952960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763110.404251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048952960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763110.404257:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763110.404260:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763110.404263:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048952960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763110.404267:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763110.404283:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763110.404289:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763110/real 1789763110] req@ffff9811c2e2a300 x1876700048953088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763110.404297:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048953088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763110.404303:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763110.404305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763110.404307:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763110.404310:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763110.404320:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763110.404326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763110/real 1789763110] req@ffff9811c2e2b800 x1876700048953216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763110.404351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048953216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763110.404358:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763110.404360:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763110.404362:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763110.659591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763111.299652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763111.683555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763112.323646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763112.707554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763113.347624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763113.731642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763114.371736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763114.755612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763115.395643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763115.523615:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763115.523627:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763115.523633:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763115.523645:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2565 00000100:00080000:3.0:1789763115.523660:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763115.523669:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763115.523716:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048953344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763115.523755:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763115.523772:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763115.523774:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763115.523779:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2565 00000100:00080000:3.0:1789763115.523784:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763115.523790:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763115.523803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763115.523820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048953472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763115.523836:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763115.523862:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763115.523866:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763115.523900:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048953344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763115.523901:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2565 00000100:00080000:3.0:1789763115.523917:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763115.523928:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763115.523935:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763115.523956:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048953600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763115.523974:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763115.523979:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763115.523982:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763115.523987:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2565 00000100:00080000:3.0:1789763115.523992:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763115.523997:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763115.524004:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763115.524015:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048953728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763115.524031:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763115.524043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763115.524047:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763115.524052:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763115.524061:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2565 00000100:00080000:3.0:1789763115.524066:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763115.524072:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763115.524080:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763115.524093:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048953856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763115.524159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763115/real 1789763115] req@ffff9811e1a21c00 x1876700048953344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763115.524192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048953344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763115.524209:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763115.524216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763115.524221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048953344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763115.524233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048953472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763115.524271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763115.524289:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763115/real 1789763115] req@ffff9811e1a22680 x1876700048953472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763115.524305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048953472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763115.524343:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763115.524352:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763115.524359:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048953472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763115.524411:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763115.524455:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763115.524562:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763115/real 1789763115] req@ffff9811e1a23480 x1876700048953600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763115.524596:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048953600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763115.524607:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763115.524612:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763115.524617:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763115.524626:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763115.524658:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763115.524672:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763115/real 1789763115] req@ffff9811e1a21500 x1876700048953728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763115.524687:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048953728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763115.524696:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763115.524700:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763115.524715:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763115.524724:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763115.524752:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763115.524764:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763115/real 1789763115] req@ffff9811e1a20700 x1876700048953856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763115.524779:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048953856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763115.524789:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763115.524792:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763115.524796:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048953856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763115.779608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763116.419593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763116.803607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763117.443630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763117.827584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763118.467699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763118.851576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763119.491676:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763119.875580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763120.515732:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763120.643672:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763120.643682:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763120.643688:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763120.643702:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2570 00000100:00080000:0.0:1789763120.643717:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763120.643727:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763120.643782:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048953984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763120.643827:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763120.643843:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763120.643845:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763120.643851:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2570 00000100:00080000:0.0:1789763120.643856:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763120.643861:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763120.643882:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763120.643888:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048953984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763120.643903:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048954112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763120.643919:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763120.643923:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763120.643926:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763120.643931:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2570 00000100:00080000:0.0:1789763120.643936:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763120.643940:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763120.643945:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763120.643955:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048954240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763120.643965:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763120.643968:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763120.643970:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763120.643973:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2570 00000100:00080000:0.0:1789763120.644037:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763120.644041:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763120.644045:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763120.644052:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763120.644057:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048954368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763120.644067:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763120.644071:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763120.644079:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763120.644082:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2570 00000100:00080000:0.0:1789763120.644086:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763120.644090:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763120.644094:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763120.644104:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048954496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763120.644292:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763120/real 1789763120] req@ffff9811c2e29f80 x1876700048953984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763120.644318:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048953984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763120.644337:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763120.644362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763120.644372:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048953984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763120.644435:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048954112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763120.644476:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763120.644555:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763120/real 1789763120] req@ffff9811c2e28a80 x1876700048954112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763120.644590:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048954112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763120.644630:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763120.644636:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763120.644643:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048954112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763120.644663:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763120.644712:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763120.644737:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763120/real 1789763120] req@ffff9811c2e28380 x1876700048954240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763120.644764:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048954240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763120.644793:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763120.644823:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763120.644847:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763120.644867:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763120.644891:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763120.644902:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763120/real 1789763120] req@ffff9811c2e2aa00 x1876700048954368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763120.644912:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048954368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763120.644919:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763120.644922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763120.644925:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763120.644929:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763120.644943:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763120.644951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763120/real 1789763120] req@ffff9811c2e2b100 x1876700048954496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763120.644970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048954496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763120.644977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763120.644980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763120.644982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763120.899596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763121.539647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763121.923565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763122.563736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763122.947562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763123.587630:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763123.971589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763124.611787:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763124.995585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763125.635654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763125.763607:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763125.763620:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763125.763625:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763125.763637:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2575 00000100:00080000:0.0:1789763125.763650:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763125.763659:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763125.763725:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048954624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763125.763814:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763125.763821:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763125.763826:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763125.763834:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2575 00000100:00080000:0.0:1789763125.763842:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763125.763850:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763125.763866:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763125.763887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048954624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763125.763901:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048954752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763125.763969:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763125.763976:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763125.763979:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763125.763990:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2575 00000100:00080000:0.0:1789763125.763998:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763125.764003:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763125.764011:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763125.764029:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048954880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763125.764042:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763125.764046:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763125.764049:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763125.764054:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2575 00000100:00080000:0.0:1789763125.764058:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763125.764065:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763125.764074:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763125.764103:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048955008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763125.764118:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763125.764128:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789763125.764129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048954752:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763125.764131:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763125.764150:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2575 00000100:00080000:0.0:1789763125.764166:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763125.764172:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763125.764182:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763125.764237:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048955136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763125.764246:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763125.764338:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763125.764395:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763126.019564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763126.659685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763127.043616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763127.683717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763128.067610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763128.707628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763128.835915:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789763128.835982:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763128.836000:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763128.836048:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763128.836055:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763128.836062:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763128.836070:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789763128.836095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763125/real 1789763128] req@ffff9811c2e29c00 x1876700048955008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763128.836125:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048955008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763128.836139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763128.836146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763128.836150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763128.836191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763125/real 1789763128] req@ffff9811c2e29880 x1876700048955136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763128.836207:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048955136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763128.836217:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763128.836222:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763128.836226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763128.836234:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763125/real 1789763128] req@ffff9811c2e29180 x1876700048954752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763128.836250:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048954752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763128.836266:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763128.836271:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763128.836276:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048954752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763128.836298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763125/real 1789763128] req@ffff9811c2e2a680 x1876700048954880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763128.836312:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048954880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763128.836321:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763128.836325:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763128.836329:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048954880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763128.836350:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763125/real 1789763128] req@ffff9811c2e2bb80 x1876700048954624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763128.836365:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048954624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763128.836375:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763128.836378:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763128.836387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048954624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763129.091607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763129.731665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763130.115642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763130.755725:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763130.883750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763130.883794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763130.883799:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763130.883812:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2580 00000100:00080000:1.0:1789763130.883826:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763130.883857:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763130.883904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048955264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763130.883937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763130.883942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763130.883944:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763130.883948:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2580 00000100:00080000:1.0:1789763130.883952:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763130.883957:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763130.883970:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763130.883986:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048955392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763130.883992:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048955264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763130.884009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763130.884014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763130.884016:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763130.884021:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2580 00000100:00080000:1.0:1789763130.884024:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763130.884029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763130.884035:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763130.884054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048955520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763130.884070:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763130.884075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763130.884088:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763130.884094:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2580 00000100:00080000:1.0:1789763130.884115:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763130.884125:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789763130.884132:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763130.884140:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763130.884166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048955648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763130.884183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763130.884189:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763130.884192:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763130.884199:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2580 00000100:00080000:1.0:1789763130.884206:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763130.884207:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763130/real 1789763130] req@ffff9811c92c5f80 x1876700048955264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763130.884218:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763130.884233:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763130.884258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048955264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763130.884260:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048955776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763130.884309:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763130.884319:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763130.884327:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048955264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763130.884372:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048955392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763130.884411:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763130.884432:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763130/real 1789763130] req@ffff9811c92c4a80 x1876700048955392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763130.884453:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048955392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763130.884528:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763130.884534:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763130.884540:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048955392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763130.884550:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763130.884587:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763130.884607:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763130/real 1789763130] req@ffff9811c92c4380 x1876700048955520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763130.884750:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048955520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763130.884765:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763130.884770:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763130.884775:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763130.884785:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763130.884819:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763130.884837:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763130/real 1789763130] req@ffff9811c92c6680 x1876700048955648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763130.884854:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048955648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763130.884865:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763130.884870:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763130.884885:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763130.884895:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763130.884927:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763130.884944:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763130/real 1789763130] req@ffff9811c92c5c00 x1876700048955776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763130.884962:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048955776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763130.884974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763130.884978:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763130.884984:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048955776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763131.139655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763131.779747:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763132.163596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763132.803736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763133.187561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763133.827653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763134.211613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763134.851720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763135.235572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763135.875560:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763136.003576:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763136.003584:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763136.003587:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763136.003596:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2585 00000100:00080000:0.0:1789763136.003606:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763136.003624:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763136.003657:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048955904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763136.003679:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763136.003682:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763136.003684:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763136.003687:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2585 00000100:00080000:0.0:1789763136.003702:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763136.003705:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763136.003714:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763136.003734:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048956032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763136.003743:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763136.003758:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763136.003760:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048955904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763136.003761:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763136.003769:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2585 00000100:00080000:0.0:1789763136.003772:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763136.003774:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763136.003778:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763136.003788:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048956160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763136.003797:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763136.003800:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763136.003801:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763136.003804:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2585 00000100:00080000:0.0:1789763136.003807:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763136.003810:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763136.003824:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763136.003835:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048956288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763136.003836:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763136.003845:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763136.003849:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763136.003850:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763136.003853:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2585 00000100:00080000:0.0:1789763136.003856:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763136.003859:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763136.003878:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763136/real 1789763136] req@ffff9811c2e28700 x1876700048955904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763136.003880:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763136.003894:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048956416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763136.003907:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048955904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763136.003930:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763136.003933:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763136.003937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048955904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763136.003943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048956032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763136.003970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763136.003978:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763136/real 1789763136] req@ffff9811c2e28e00 x1876700048956032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763136.003986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048956032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763136.003992:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763136.003994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763136.003996:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048956032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763136.004014:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763136.004025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763136.004031:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763136/real 1789763136] req@ffff9811c2e2ad80 x1876700048956160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763136.004038:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048956160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763136.004044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763136.004045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763136.004047:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763136.004050:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763136.004061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763136.004066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763136/real 1789763136] req@ffff9811c2e2bb80 x1876700048956288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763136.004083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048956288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763136.004088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763136.004090:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763136.004092:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763136.004096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763136.004107:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763136.004113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763136/real 1789763136] req@ffff9811c2e2a680 x1876700048956416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763136.004129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048956416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763136.004134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763136.004136:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763136.004138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763136.259563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763136.899649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763137.283592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763137.923628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763138.307580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763138.947719:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763139.331607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763139.971713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763140.355586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763140.995643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763141.123587:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763141.123600:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763141.123605:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763141.123619:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2590 00000100:00080000:0.0:1789763141.123633:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763141.123643:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763141.123721:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048956544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763141.123765:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763141.123770:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763141.123773:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763141.123778:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2590 00000100:00080000:0.0:1789763141.123784:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763141.123789:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763141.123800:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763141.123818:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048956672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763141.123835:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763141.123840:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763141.123843:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763141.123852:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048956544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763141.123853:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2590 00000100:00080000:0.0:1789763141.123882:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763141.123889:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763141.123898:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763141.123916:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048956800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763141.123931:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763141.123935:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763141.123937:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763141.123944:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2590 00000800:00000100:3.0:1789763141.123953:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763141.123955:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763141.123964:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763141.123972:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763141.123988:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048956928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763141.124001:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763141.124005:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763141.124008:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763141.124014:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2590 00000100:00080000:0.0:1789763141.124018:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763141.124022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763141/real 1789763141] req@ffff9811c2e29180 x1876700048956544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763141.124027:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763141.124040:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763141.124066:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048957056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763141.124074:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048956544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763141.124127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763141.124134:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763141.124142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048956544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763141.124205:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048956672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763141.124255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763141.124282:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763141/real 1789763141] req@ffff9811c2e29880 x1876700048956672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763141.124309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048956672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763141.124324:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763141.124329:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763141.124333:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048956672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763141.124342:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763141.124371:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763141.124386:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763141/real 1789763141] req@ffff9811c2e29c00 x1876700048956800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763141.124401:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048956800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763141.124410:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763141.124413:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763141.124417:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763141.124435:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763141.124519:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763141.124555:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763141/real 1789763141] req@ffff9811c2e2b100 x1876700048956928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763141.124617:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048956928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763141.124629:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763141.124632:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763141.124637:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048956928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763141.124645:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763141.124709:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763141.124730:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763141/real 1789763141] req@ffff9811c2e2aa00 x1876700048957056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763141.124746:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048957056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763141.124756:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763141.124760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763141.124763:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763141.379640:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763142.019745:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763142.403587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763143.043662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763143.427551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763144.067712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763144.451603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763145.091695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763145.475591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763146.115661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763146.243561:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763146.243582:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763146.243590:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763146.243610:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2595 00000100:00080000:0.0:1789763146.243660:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763146.243673:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763146.243744:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048957184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763146.243786:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763146.243791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763146.243794:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763146.243799:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2595 00000100:00080000:0.0:1789763146.243804:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763146.243809:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763146.243823:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763146.243839:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048957312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763146.243857:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763146.243862:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763146.243865:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763146.243887:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2595 00000100:00100000:3.0:1789763146.243890:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048957184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763146.243891:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763146.243904:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763146.243933:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763146.243958:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048957440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763146.243974:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763146.243979:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763146.243981:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763146.243986:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2595 00000100:00080000:0.0:1789763146.243992:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763146.243997:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763146.244004:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763146.244028:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048957568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763146.244032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763146.244050:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763146.244055:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763146.244059:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763146.244065:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2595 00000100:00080000:0.0:1789763146.244073:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763146.244079:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763146.244088:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763146.244126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763146/real 1789763146] req@ffff9811c2e28380 x1876700048957184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763146.244129:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048957696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763146.244190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048957184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763146.244209:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763146.244216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763146.244223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048957184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763146.244233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048957312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763146.244281:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763146.244299:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763146/real 1789763146] req@ffff9811c2e28a80 x1876700048957312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763146.244315:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048957312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763146.244333:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763146.244337:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763146.244342:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048957312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763146.244377:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763146.244415:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763146.244439:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763146/real 1789763146] req@ffff9811c2e29f80 x1876700048957440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763146.244533:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048957440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763146.244545:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763146.244549:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763146.244553:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763146.244562:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763146.244596:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763146.244652:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763146/real 1789763146] req@ffff9811c2e2b800 x1876700048957568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763146.244669:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048957568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763146.244678:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763146.244682:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763146.244686:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763146.244694:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763146.244728:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763146.244740:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763146/real 1789763146] req@ffff9811c2e2a300 x1876700048957696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763146.244754:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048957696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763146.244783:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763146.244787:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763146.244792:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048957696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763146.499663:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763147.139665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763147.523702:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763148.163683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763148.547661:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763149.187710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763149.571627:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763150.211688:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763150.595627:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763151.235677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763151.363654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763151.363670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763151.363692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763151.363708:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2600 00000100:00080000:2.0:1789763151.363725:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763151.363737:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763151.363797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048957824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763151.363839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763151.363844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763151.363847:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763151.363852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2600 00000100:00080000:2.0:1789763151.363857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763151.363863:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763151.363879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763151.363898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048957952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763151.363910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048957824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763151.363923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763151.363929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763151.363932:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763151.363937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2600 00000100:00080000:2.0:1789763151.363941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763151.363946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763151.363952:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763151.363967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048958080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763151.363998:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763151.364002:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763151.364005:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763151.364010:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2600 00000100:00080000:2.0:1789763151.364017:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763151.364022:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763151.364031:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763151.364041:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763151.364053:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048958208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763151.364068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763151.364073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763151.364076:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763151.364083:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2600 00000100:00080000:2.0:1789763151.364088:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763151.364092:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763151.364098:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763151.364112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048958336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763151.364139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763151/real 1789763151] req@ffff9811d8638700 x1876700048957824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763151.364185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048957824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763151.364203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763151.364210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763151.364219:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048957824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763151.364266:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048957952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763151.364319:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763151.364347:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763151/real 1789763151] req@ffff9811d8638e00 x1876700048957952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763151.364367:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048957952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763151.364400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763151.364408:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763151.364416:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048957952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763151.364430:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763151.364544:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763151.364597:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763151/real 1789763151] req@ffff9811d8638380 x1876700048958080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763151.364628:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048958080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763151.364647:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763151.364652:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763151.364657:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763151.364667:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763151.364715:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763151.364741:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763151/real 1789763151] req@ffff9811d8638000 x1876700048958208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763151.364768:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048958208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763151.364785:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763151.364791:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763151.364815:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763151.364829:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763151.364875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763151.364899:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763151/real 1789763151] req@ffff9811d863aa00 x1876700048958336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763151.364927:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048958336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763151.364944:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763151.364949:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763151.364953:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763151.619672:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763152.259678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763152.643661:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763153.283634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763153.667676:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763154.307661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763154.691691:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763155.331674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763155.715651:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763156.355706:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763156.483585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763156.483607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763156.483612:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763156.483628:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2606 00000100:00080000:2.0:1789763156.483641:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763156.483652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763156.483696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048958464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763156.483733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763156.483743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763156.483746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763156.483754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2606 00000100:00080000:2.0:1789763156.483760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763156.483767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763156.483810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763156.483839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048958592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763156.483869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763156.483877:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048958464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763156.483878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763156.483886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763156.483900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2606 00000100:00080000:2.0:1789763156.483907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763156.483913:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763156.483919:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763156.483960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048958720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763156.483974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763156.483979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763156.483983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763156.483995:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2606 00000100:00080000:2.0:1789763156.484022:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763156.484032:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763156.484044:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763156.484046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763156.484078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048958848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763156.484099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763156.484105:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763156.484108:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763156.484119:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2606 00000100:00080000:2.0:1789763156.484124:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763156.484128:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763156.484139:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763156.484159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763156/real 1789763156] req@ffff9811d863b480 x1876700048958464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763156.484167:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048958976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763156.484599:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048958464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763156.484622:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763156.484633:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763156.484661:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048958464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763156.484715:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048958592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763156.484848:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763156.484869:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763156/real 1789763156] req@ffff9811d863a300 x1876700048958592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763156.484886:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048958592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763156.484903:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763156.484910:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763156.484916:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048958592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763156.484930:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763156.484969:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763156.484991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763156/real 1789763156] req@ffff9811d863b800 x1876700048958720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763156.485013:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048958720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763156.485023:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763156.485027:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763156.485031:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763156.485039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763156.485067:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763156.485093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763156/real 1789763156] req@ffff9811d8638a80 x1876700048958848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763156.485108:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048958848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763156.485118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763156.485122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763156.485125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763156.485132:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763156.485152:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763156.485162:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763156/real 1789763156] req@ffff9811d863a680 x1876700048958976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763156.485175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048958976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763156.485184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763156.485187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763156.485190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048958976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763156.739695:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763157.379758:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763157.763638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763158.403665:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763158.787587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763159.427684:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763159.811665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763160.451700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763160.835630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763161.475675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763161.603695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763161.603717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763161.603725:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763161.603743:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2611 00000100:00080000:1.0:1789763161.603761:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763161.603772:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763161.603832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048959104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763161.603913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763161.603919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763161.603922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763161.603928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2611 00000100:00080000:1.0:1789763161.603933:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763161.603938:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763161.603953:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763161.603971:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048959232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763161.603986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763161.603990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763161.603992:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763161.603997:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2611 00000100:00080000:1.0:1789763161.604001:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763161.604005:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763161.604012:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763161.604018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048959104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763161.604029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048959360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763161.604045:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763161.604050:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763161.604052:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763161.604057:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2611 00000100:00080000:1.0:1789763161.604071:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763161.604077:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763161.604086:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763161.604102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048959488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763161.604116:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763161.604120:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763161.604123:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763161.604126:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2611 00000100:00080000:1.0:1789763161.604131:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763161.604134:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763161.604140:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763161.604151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763161.604157:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048959616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763161.604253:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763161/real 1789763161] req@ffff9811c92c7480 x1876700048959104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763161.604283:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048959104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763161.604300:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763161.604306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763161.604312:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048959104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763161.604322:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048959232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763161.604357:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763161.604375:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763161/real 1789763161] req@ffff9811c92c4e00 x1876700048959232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763161.604390:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048959232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763161.604401:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763161.604404:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763161.604408:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048959232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763161.604446:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763161.604565:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763161.604585:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763161/real 1789763161] req@ffff9811c92c4000 x1876700048959360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763161.604602:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048959360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763161.604632:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763161.604636:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763161.604641:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763161.604649:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763161.604681:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763161.604701:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763161/real 1789763161] req@ffff9811c92c5500 x1876700048959488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763161.604725:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048959488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763161.604741:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763161.604748:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763161.604755:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763161.604768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763161.604808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763161.604825:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763161/real 1789763161] req@ffff9811c92c5180 x1876700048959616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763161.604841:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048959616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763161.604851:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763161.604855:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763161.604870:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048959616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763161.859597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763162.499696:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763162.883540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763163.523687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763163.907656:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763164.547702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763164.931620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763165.571640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763165.955598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763166.595575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763166.723614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763166.723625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763166.723630:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763166.723641:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2616 00000100:00080000:1.0:1789763166.723654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763166.723664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763166.723706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048959744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763166.723737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763166.723764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763166.723766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763166.723770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2616 00000100:00080000:1.0:1789763166.723775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763166.723780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763166.723791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763166.723805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048959872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763166.723819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763166.723821:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048959744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763166.723828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763166.723834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763166.723839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2616 00000100:00080000:1.0:1789763166.723842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763166.723846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763166.723850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763166.723861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048960000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763166.723871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763166.723875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763166.723877:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763166.723881:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2616 00000100:00080000:1.0:1789763166.723887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763166.723891:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763166.723903:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763166.723919:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763166.723921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048960128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763166.723934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763166.723938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763166.723941:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763166.723944:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2616 00000100:00080000:1.0:1789763166.723949:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763166.723953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763166.723958:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763166.723976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048960256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763166.723986:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763166/real 1789763166] req@ffff9811c92c6d80 x1876700048959744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763166.724014:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048959744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763166.724028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763166.724032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763166.724036:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048959744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763166.724060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048959872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763166.724082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763166.724094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763166/real 1789763166] req@ffff9811c92c5880 x1876700048959872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763166.724115:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048959872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763166.724124:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763166.724127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763166.724141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048959872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763166.724148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763166.724170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763166.724181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763166/real 1789763166] req@ffff9811c92c7b80 x1876700048960000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763166.724192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048960000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763166.724200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763166.724202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763166.724206:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763166.724211:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763166.724226:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763166.724234:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763166/real 1789763166] req@ffff9811c92c5180 x1876700048960128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763166.724244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048960128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763166.724251:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763166.724254:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763166.724256:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763166.724261:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763166.724275:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763166.724295:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763166/real 1789763166] req@ffff9811c92c5500 x1876700048960256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763166.724306:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048960256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763166.724313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763166.724316:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763166.724318:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763166.979655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763167.619552:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763168.003579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763168.643626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763169.027604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763169.667599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763170.051575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763170.691638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763171.075565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763171.715651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763171.843582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763171.843590:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763171.843594:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763171.843603:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2621 00000100:00080000:1.0:1789763171.843612:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763171.843619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763171.843662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048960384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763171.843689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763171.843693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763171.843695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763171.843700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2621 00000100:00080000:1.0:1789763171.843715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763171.843721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763171.843724:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048960384:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789763171.843730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763171.843745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048960512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763171.843754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763171.843757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763171.843758:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763171.843761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2621 00000100:00080000:1.0:1789763171.843764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763171.843766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763171.843769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763171.843778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048960640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763171.843785:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763171.843791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763171.843794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763171.843796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763171.843801:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2621 00000100:00080000:1.0:1789763171.843805:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763171.843807:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763171.843811:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763171.843818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048960768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763171.843825:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763171/real 1789763171] req@ffff9811c92c4000 x1876700048960384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763171.843860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763171.843863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789763171.843866:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048960384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763171.843867:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763171.843875:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2621 00000100:00080000:1.0:1789763171.843881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763171.843884:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763171.843886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763171.843891:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789763171.843892:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763171.843897:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048960384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763171.843907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048960896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763171.843920:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048960512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763171.843939:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763171.843956:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763171/real 1789763171] req@ffff9811c92c4e00 x1876700048960512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763171.843965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048960512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763171.843971:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763171.843973:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763171.843975:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048960512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763171.843979:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763171.843991:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763171.843998:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763171/real 1789763171] req@ffff9811c92c7480 x1876700048960640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763171.844005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048960640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763171.844010:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763171.844011:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763171.844013:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763171.844016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763171.844037:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763171.844044:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763171/real 1789763171] req@ffff9811c92c5c00 x1876700048960768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763171.844051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048960768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763171.844061:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763171.844063:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763171.844065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763171.844069:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763171.844083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763171.844096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763171/real 1789763171] req@ffff9811c92c6680 x1876700048960896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763171.844104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048960896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763171.844109:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763171.844111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763171.844113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048960896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763172.099596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763172.739651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763173.123645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763173.763639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763174.147676:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763174.787667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763175.171730:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763175.811647:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763176.195639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763176.835633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763176.963641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763176.963656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763176.963662:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763176.963692:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2626 00000100:00080000:1.0:1789763176.963711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763176.963723:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763176.963790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048961024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763176.963834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763176.963839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763176.963842:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763176.963848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2626 00000100:00080000:1.0:1789763176.963852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763176.963856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763176.963900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763176.963918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048961152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763176.963937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763176.963954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763176.963957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048961024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763176.963958:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763176.963970:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2626 00000100:00080000:1.0:1789763176.963981:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763176.963986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763176.963992:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763176.964004:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048961280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763176.964015:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763176.964020:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763176.964023:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763176.964027:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2626 00000100:00080000:1.0:1789763176.964031:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763176.964035:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763176.964042:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763176.964056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048961408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763176.964068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763176.964072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763176.964082:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763176.964088:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2626 00000100:00080000:1.0:1789763176.964094:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763176.964099:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789763176.964110:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763176.964112:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763176.964132:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048961536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763176.964186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763176/real 1789763176] req@ffff9811c92c4380 x1876700048961024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763176.964217:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048961024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763176.964245:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763176.964251:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763176.964257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048961024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763176.964294:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048961152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763176.964329:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763176.964346:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763176/real 1789763176] req@ffff9811c92c4a80 x1876700048961152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763176.964361:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048961152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763176.964372:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763176.964376:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763176.964380:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048961152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763176.964415:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763176.964450:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763176.964509:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763176/real 1789763176] req@ffff9811c92c5f80 x1876700048961280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763176.964526:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048961280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763176.964537:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763176.964541:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763176.964545:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763176.964565:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763176.964598:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763176.964613:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763176/real 1789763176] req@ffff9811c92c6a00 x1876700048961408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763176.964628:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048961408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763176.964638:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763176.964641:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763176.964645:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763176.964652:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763176.964672:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763176.964683:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763176/real 1789763176] req@ffff9811c92c6300 x1876700048961536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763176.964696:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048961536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763176.964705:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763176.964709:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763176.964712:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763177.219657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763177.859690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763178.243641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763178.883648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763179.267562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763179.907638:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763180.291602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763180.931626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763181.315577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763181.955645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763182.083579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763182.083592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763182.083598:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763182.083611:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2631 00000100:00080000:1.0:1789763182.083625:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763182.083635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763182.083702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048961664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763182.083751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763182.083759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763182.083763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763182.083772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2631 00000100:00080000:1.0:1789763182.083780:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763182.083788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763182.083805:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763182.083839:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048961792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763182.083887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763182.083897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763182.083899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048961664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763182.083901:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763182.083918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2631 00000100:00080000:1.0:1789763182.083926:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763182.083933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763182.083942:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763182.083963:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048961920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763182.083977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763182.083983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763182.083986:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763182.083990:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2631 00000100:00080000:1.0:1789763182.083996:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763182.084004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763182.084012:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763182.084021:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763182.084040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048962048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763182.084053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763182.084058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763182.084060:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763182.084067:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2631 00000100:00100000:3.0:1789763182.084075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763182/real 1789763182] req@ffff9811c92c7100 x1876700048961664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763182.084101:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763182.084107:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048961664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763182.084116:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763182.084134:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763182.084140:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763182.084148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763182.084156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048962176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763182.084159:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048961664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763182.084202:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048961792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763182.084237:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763182.084255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763182/real 1789763182] req@ffff9811c92c7800 x1876700048961792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763182.084270:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048961792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763182.084280:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763182.084284:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763182.084288:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048961792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763182.084294:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763182.084313:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763182.084325:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763182/real 1789763182] req@ffff9811c92c4700 x1876700048961920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763182.084388:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048961920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763182.084406:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763182.084412:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763182.084419:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048961920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763182.084432:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763182.084525:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763182.084544:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763182/real 1789763182] req@ffff9811c92c6300 x1876700048962048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763182.084560:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048962048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763182.084570:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763182.084574:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763182.084578:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763182.084585:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763182.084610:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763182.084621:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763182/real 1789763182] req@ffff9811c92c6a00 x1876700048962176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763182.084635:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048962176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763182.084666:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763182.084670:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763182.084675:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763182.339571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763182.979643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763183.363591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763184.003642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763184.387622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763185.027620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763185.411639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763186.051642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763186.435582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763187.075660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763187.203621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763187.203637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763187.203643:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763187.203659:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2636 00000100:00080000:3.0:1789763187.203676:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763187.203688:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763187.203764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048962304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763187.203807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763187.203813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763187.203815:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763187.203821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2636 00000100:00080000:3.0:1789763187.203828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763187.203837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763187.203857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763187.203887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048962432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763187.203895:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048962304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763187.203942:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763187.203950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763187.203955:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763187.203985:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2636 00000100:00080000:3.0:1789763187.203997:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763187.204006:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763187.204012:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763187.204030:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763187.204062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048962560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763187.204082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763187/real 1789763187] req@ffff9811e1a23100 x1876700048962304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763187.204085:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763187.204107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763187.204117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763187.204130:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2636 00000100:00100000:1.0:1789763187.204135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048962304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763187.204142:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763187.204161:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:1.0:1789763187.204162:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763187.204178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:3.0:1789763187.204182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763187.204187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048962304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763187.204232:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048962432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763187.204235:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048962688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763187.204265:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763187.204271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763187.204282:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763187.204288:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763187.204298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763187/real 1789763187] req@ffff9811e1a23800 x1876700048962432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763187.204300:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2636 00000100:00100000:1.0:1789763187.204346:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048962432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763187.204352:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763187.204361:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763187.204373:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763187.204374:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763187.204390:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763187.204396:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048962432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763187.204411:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763187.204431:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048962816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763187.204451:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763187.204513:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763187/real 1789763187] req@ffff9811e1a20380 x1876700048962560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763187.204535:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048962560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763187.204547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763187.204554:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763187.204559:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763187.204583:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763187.204610:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763187.204623:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763187/real 1789763187] req@ffff9811e1a20e00 x1876700048962688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763187.204638:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048962688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763187.204647:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763187.204651:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763187.204655:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763187.204662:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763187.204695:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763187.204722:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763187/real 1789763187] req@ffff9811e1a20000 x1876700048962816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763187.204737:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048962816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763187.204748:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763187.204751:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763187.204755:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048962816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763187.459582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763188.099656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763188.483583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763189.123669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763189.507566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763190.147675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763190.531577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763191.171701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763191.555611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763192.195733:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763192.323589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763192.323606:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763192.323613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763192.323631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2641 00000100:00080000:3.0:1789763192.323649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763192.323664:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763192.323725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048962944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763192.323781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763192.323788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763192.323790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763192.323796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2641 00000100:00080000:3.0:1789763192.323801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763192.323807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763192.323846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763192.323871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048963072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763192.323889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763192.323895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763192.323898:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763192.323906:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2641 00000100:00080000:3.0:1789763192.323917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763192.323922:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048962944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763192.323933:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763192.323948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763192.323979:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048963200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763192.324005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763192.324013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763192.324019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763192.324028:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2641 00000100:00080000:3.0:1789763192.324036:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763192.324044:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763192.324104:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048963072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763192.324171:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963200:192.168.203.113@tcp:38:kworker.0 00000008:00100000:3.0:1789763192.324386:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763192.324413:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048963328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763192.324439:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763192.324444:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763192.324495:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763192.324503:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2641 00000100:00080000:3.0:1789763192.324512:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763192.324523:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963328:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763192.324524:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763192.324535:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763192.324562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048963456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763192.324622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763192.579595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763193.219697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763193.603583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763194.243693:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763194.627552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763195.267704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763195.395983:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789763195.396059:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763195.396121:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763195.396180:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763195.396191:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763195.396211:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763195.396220:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789763195.396352:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763192/real 1789763195] req@ffff9811e1a20380 x1876700048963456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763195.396419:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048963456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763195.396442:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763195.396452:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763195.396520:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763195.396540:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763192/real 1789763195] req@ffff9811e1a20e00 x1876700048963328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763195.396587:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048963328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763195.396605:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763195.396611:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763195.396617:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763195.396631:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763192/real 1789763195] req@ffff9811e1a20000 x1876700048963200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763195.396656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048963200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763195.396672:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763195.396697:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763195.396704:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763195.396718:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763192/real 1789763195] req@ffff9811e1a21f80 x1876700048962944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763195.396747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048962944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763195.396763:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763195.396770:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763195.396777:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048962944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763195.396790:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763192/real 1789763195] req@ffff9811e1a20a80 x1876700048963072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763195.396815:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048963072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763195.396831:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763195.396838:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763195.396844:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048963072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763195.651620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763196.291606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763196.675554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763197.315681:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763197.443735:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763197.443748:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763197.443768:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763197.443785:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2647 00000100:00080000:0.0:1789763197.443802:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763197.443814:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763197.443871:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048963584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763197.443911:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763197.443919:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763197.443923:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763197.443930:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2647 00000100:00080000:0.0:1789763197.443938:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763197.443946:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763197.443966:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763197.444006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048963584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763197.444014:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048963712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763197.444044:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763197.444052:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763197.444084:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763197.444095:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2647 00000100:00080000:0.0:1789763197.444106:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763197.444115:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763197.444128:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763197.444147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763197.444161:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048963840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763197.444187:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763197.444194:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763197.444198:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763197.444207:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2647 00000100:00080000:0.0:1789763197.444217:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763197.444226:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763197.444273:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763197/real 1789763197] req@ffff9811c2e28000 x1876700048963584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763197.444274:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763197.444312:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048963968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763197.444330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048963584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763197.444342:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763197.444360:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763197.444366:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763197.444388:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2647 00000100:00080000:2.0:1789763197.444400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763197.444402:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763197.444423:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763197.444425:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763197.444445:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763197.444445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048963584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763197.444572:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048964096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763197.444575:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048963712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763197.444652:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763197.444684:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763197/real 1789763197] req@ffff9811c2e29500 x1876700048963712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763197.444712:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048963712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763197.444732:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763197.444741:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763197.444749:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048963712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763197.444774:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763197.444818:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763197.444844:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763197/real 1789763197] req@ffff9811c2e2b480 x1876700048963840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763197.444871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048963840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763197.444889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763197.444898:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763197.444905:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763197.444933:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763197.444981:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763197.445009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763197/real 1789763197] req@ffff9811c2e2a300 x1876700048963968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763197.445037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048963968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763197.445054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763197.445061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763197.445068:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048963968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763197.445103:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763197.445147:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763197.445171:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763197/real 1789763197] req@ffff9811c2e2b800 x1876700048964096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763197.445207:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048964096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763197.445222:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763197.445227:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763197.445233:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763197.699593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763198.339705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763198.723596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763199.363697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763199.747537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763200.387631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763200.771544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763201.411646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763201.795528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763202.435736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763202.563661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763202.563677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763202.563684:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763202.563701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2652 00000100:00080000:3.0:1789763202.563763:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763202.563789:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763202.563860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048964224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763202.563904:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763202.563909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763202.563912:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763202.563918:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2652 00000100:00080000:3.0:1789763202.563922:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763202.563927:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763202.563943:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763202.563961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048964352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763202.563977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763202.563981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763202.563984:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763202.563988:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2652 00000100:00080000:3.0:1789763202.563991:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763202.563997:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763202.564003:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763202.564028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048964480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763202.564043:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763202.564059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048964224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763202.564061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763202.564067:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763202.564085:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2652 00000100:00080000:3.0:1789763202.564090:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763202.564103:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763202.564111:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763202.564142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048964608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763202.564156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763202.564161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763202.564163:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763202.564167:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2652 00000100:00080000:3.0:1789763202.564173:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763202.564179:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763202.564185:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763202.564224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048964736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763202.564232:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763202.564320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763202/real 1789763202] req@ffff9811e1a23800 x1876700048964224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763202.564364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048964224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763202.564380:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763202.564385:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763202.564391:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048964224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763202.564402:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048964352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763202.564435:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763202.564451:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763202/real 1789763202] req@ffff9811e1a23100 x1876700048964352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763202.564519:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048964352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763202.564531:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763202.564535:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763202.564539:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048964352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763202.564575:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763202.564603:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763202.564616:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763202/real 1789763202] req@ffff9811e1a20700 x1876700048964480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763202.564631:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048964480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763202.564659:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763202.564663:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763202.564668:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763202.564676:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763202.564702:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763202.564714:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763202/real 1789763202] req@ffff9811e1a21500 x1876700048964608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763202.564729:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048964608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763202.564739:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763202.564752:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763202.564755:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763202.564762:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763202.564780:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763202.564790:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763202/real 1789763202] req@ffff9811e1a23480 x1876700048964736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763202.564803:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048964736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763202.564812:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763202.564826:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763202.564838:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048964736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763202.819547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763203.459684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763203.843546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763204.483686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763204.867592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763205.507679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763205.891578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763206.531609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763206.915594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763207.555677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763207.683674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763207.683686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763207.683692:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763207.683705:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2657 00000100:00080000:0.0:1789763207.683719:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763207.683728:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763207.683778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048964864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763207.683831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763207.683836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763207.683839:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763207.683844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2657 00000100:00080000:0.0:1789763207.683849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763207.683853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763207.683864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763207.683883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048964992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763207.683899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763207.683904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763207.683907:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763207.683916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2657 00000100:00100000:2.0:1789763207.683918:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048964864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763207.683920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763207.683948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763207.683956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763207.683978:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048965120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763207.683994:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763207.683999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763207.684002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763207.684007:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2657 00000800:00000100:2.0:1789763207.684015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763207.684020:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763207.684030:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763207.684038:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763207.684056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048965248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763207.684069:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763207.684074:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763207.684077:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763207.684083:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763207/real 1789763207] req@ffff9811c2e29f80 x1876700048964864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763207.684088:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2657 00000100:00080000:0.0:1789763207.684100:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763207.684112:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763207.684120:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763207.684129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048964864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763207.684151:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048965376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763207.684189:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763207.684209:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763207.684217:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048964864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763207.684231:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048964992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763207.684267:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763207.684284:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763207/real 1789763207] req@ffff9811c2e28a80 x1876700048964992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763207.684300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048964992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763207.684311:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763207.684315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763207.684319:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048964992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763207.684367:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763207.684408:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763207.684432:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763207/real 1789763207] req@ffff9811c2e28380 x1876700048965120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763207.684513:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048965120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763207.684527:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763207.684532:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763207.684537:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763207.684545:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763207.684584:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763207.684608:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763207/real 1789763207] req@ffff9811c2e2aa00 x1876700048965248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763207.684634:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048965248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763207.684652:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763207.684659:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763207.684665:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763207.684674:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763207.684704:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763207.684718:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763207/real 1789763207] req@ffff9811c2e2b100 x1876700048965376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763207.684746:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048965376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763207.684756:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763207.684760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763207.684764:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763207.939608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763208.579754:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763208.963591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763209.603602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763209.987556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763210.627676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763211.011566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763211.651703:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763212.035615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763212.675674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763212.803605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763212.803642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763212.803647:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763212.803660:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2662 00000100:00080000:0.0:1789763212.803675:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763212.803684:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763212.803730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048965504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763212.803771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763212.803777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763212.803779:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763212.803784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2662 00000100:00080000:0.0:1789763212.803789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763212.803793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763212.803804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763212.803819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048965632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763212.803835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763212.803841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763212.803843:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763212.803849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2662 00000100:00080000:0.0:1789763212.803853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763212.803858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763212.803865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763212.803877:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048965504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763212.803883:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048965760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763212.803910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763212.803915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763212.803918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763212.803924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2662 00000100:00080000:0.0:1789763212.803930:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763212.803936:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763212.803943:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763212.803960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048965888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763212.803975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763212.803982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763212.803989:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763212.803992:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763212.803998:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2662 00000100:00080000:0.0:1789763212.804016:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763212.804021:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763212.804029:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763212.804043:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763212/real 1789763212] req@ffff9811c2e29c00 x1876700048965504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763212.804048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048966016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763212.804112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048965504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763212.804156:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763212.804163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763212.804169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048965504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763212.804179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048965632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763212.804213:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763212.804230:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763212/real 1789763212] req@ffff9811c2e29880 x1876700048965632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763212.804246:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048965632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763212.804270:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763212.804275:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763212.804279:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048965632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763212.804310:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763212.804357:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763212.804379:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763212/real 1789763212] req@ffff9811c2e29180 x1876700048965760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763212.804394:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048965760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763212.804405:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763212.804409:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763212.804413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763212.804420:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763212.804441:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763212.804452:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763212/real 1789763212] req@ffff9811c2e2a680 x1876700048965888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763212.804516:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048965888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763212.804527:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763212.804530:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763212.804545:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048965888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763212.804554:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763212.804592:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763212.804615:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763212/real 1789763212] req@ffff9811c2e2bb80 x1876700048966016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763212.804641:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048966016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763212.804659:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763212.804665:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763212.804673:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763213.059621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763213.699696:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763214.083587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763214.723656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763215.107621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763215.747631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763216.131665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763216.771687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763217.155579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763217.795697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763217.923584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763217.923597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763217.923603:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763217.923616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2667 00000100:00080000:3.0:1789763217.923630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763217.923651:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763217.923699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048966144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763217.923735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763217.923740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763217.923742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763217.923747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2667 00000100:00080000:3.0:1789763217.923752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763217.923756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763217.923799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763217.923819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048966272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763217.923837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763217.923843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763217.923845:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763217.923850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2667 00000100:00080000:3.0:1789763217.923856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763217.923867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763217.923870:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048966144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763217.923877:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763217.923906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048966400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763217.923927:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763217.923933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763217.923936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763217.923941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2667 00000100:00080000:3.0:1789763217.923948:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763217.923953:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763217.923960:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763217.923974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048966528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763217.923987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763217.923991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1789763217.924029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763217.924042:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763217.924053:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2667 00000100:00080000:3.0:1789763217.924064:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763217.924075:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763217.924108:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763217.924138:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048966656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763217.924147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763217/real 1789763217] req@ffff9811e1a22680 x1876700048966144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763217.924199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048966144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763217.924219:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763217.924228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763217.924234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048966144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763217.924270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048966272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763217.924311:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763217.924328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763217/real 1789763217] req@ffff9811e1a21c00 x1876700048966272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763217.924345:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048966272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763217.924356:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763217.924360:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763217.924365:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048966272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763217.924404:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763217.924427:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763217.924453:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763217/real 1789763217] req@ffff9811e1a21880 x1876700048966400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763217.924531:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048966400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763217.924544:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763217.924562:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763217.924574:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763217.924589:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763217.924625:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763217.924648:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763217/real 1789763217] req@ffff9811e1a22300 x1876700048966528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763217.924692:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048966528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763217.924709:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763217.924715:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763217.924722:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763217.924735:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763217.924781:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763217.924798:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763217/real 1789763217] req@ffff9811e1a21180 x1876700048966656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763217.924814:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048966656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763217.924825:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763217.924829:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763217.924833:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048966656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763218.179621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763218.819722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763219.203604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763219.843658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763220.227686:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763220.867735:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763221.251661:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763221.891691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763222.275629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763222.915764:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763223.043618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763223.043631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763223.043636:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763223.043648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2672 00000100:00080000:3.0:1789763223.043661:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763223.043670:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763223.043716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048966784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763223.043750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763223.043754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763223.043757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763223.043761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2672 00000100:00080000:3.0:1789763223.043764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763223.043787:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763223.043800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763223.043817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048966912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763223.043834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763223.043841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763223.043848:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763223.043851:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048966784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763223.043858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2672 00000100:00080000:3.0:1789763223.043870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763223.043877:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763223.043882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763223.043899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048967040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763223.043910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763223.043935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763223.043937:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763223.043943:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2672 00000100:00080000:3.0:1789763223.043970:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763223.043977:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763223.043984:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763223.043995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763223.044002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048967168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763223.044019:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763223.044031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763223.044035:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763223.044044:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2672 00000100:00080000:3.0:1789763223.044074:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763223.044081:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763223.044108:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763223.044128:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048967296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763223.044140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763223/real 1789763223] req@ffff9811e1a22a00 x1876700048966784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763223.044199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048966784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763223.044238:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763223.044246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763223.044256:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048966784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763223.044299:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048966912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763223.044338:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763223.044373:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763223/real 1789763223] req@ffff9811e1a22d80 x1876700048966912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763223.044389:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048966912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763223.044400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763223.044405:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763223.044409:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048966912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763223.044417:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763223.044442:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763223.044540:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763223/real 1789763223] req@ffff9811e1a21180 x1876700048967040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763223.044568:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048967040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763223.044579:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763223.044584:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763223.044588:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763223.044622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763223.044656:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763223.044672:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763223/real 1789763223] req@ffff9811e1a22300 x1876700048967168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763223.044687:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048967168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763223.044698:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763223.044702:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763223.044706:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763223.044713:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763223.044732:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763223.044742:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763223/real 1789763223] req@ffff9811e1a21880 x1876700048967296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763223.044755:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048967296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763223.044765:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763223.044768:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763223.044772:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763223.299602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763223.939656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763224.323602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763224.963746:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763225.347589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763225.987642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763226.371624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763227.011768:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763227.395596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763228.035575:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763228.163596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763228.163609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763228.163615:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763228.163628:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2677 00000100:00080000:3.0:1789763228.163653:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763228.163663:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763228.163711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048967424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763228.163747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763228.163752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763228.163754:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763228.163760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2677 00000100:00080000:3.0:1789763228.163785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763228.163793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763228.163818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763228.163839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048967552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763228.163853:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048967424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763228.163861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763228.163866:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763228.163878:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763228.163885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2677 00000100:00080000:3.0:1789763228.163888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763228.163892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763228.163914:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763228.163929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048967680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763228.163948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763228.163954:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763228.163956:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763228.163961:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2677 00000100:00080000:3.0:1789763228.163968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763228.163974:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763228.163980:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763228.163995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048967808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763228.164049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763228.164054:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763228.164057:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763228.164063:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2677 00000100:00080000:3.0:1789763228.164069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763228.164083:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789763228.164085:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763228.164109:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763228.164133:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048967936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763228.164183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763228/real 1789763228] req@ffff9811e1a21c00 x1876700048967424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763228.164217:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048967424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763228.164236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763228.164245:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763228.164250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048967424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763228.164292:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048967552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763228.164333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763228.164367:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763228/real 1789763228] req@ffff9811e1a22680 x1876700048967552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763228.164378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048967552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763228.164384:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763228.164387:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763228.164390:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048967552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763228.164395:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763228.164440:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763228.164466:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763228/real 1789763228] req@ffff9811e1a23480 x1876700048967680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763228.164573:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048967680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763228.164586:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763228.164596:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763228.164608:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763228.164628:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763228.164691:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763228.164727:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763228/real 1789763228] req@ffff9811e1a21500 x1876700048967808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763228.164736:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048967808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763228.164742:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763228.164744:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763228.164746:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763228.164750:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763228.164765:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763228.164771:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763228/real 1789763228] req@ffff9811e1a20700 x1876700048967936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763228.164778:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048967936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763228.164795:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763228.164797:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763228.164799:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048967936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763228.419587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763229.059749:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763229.443574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763230.083679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763230.467593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763231.107736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763231.491587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763232.131625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763232.515615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763233.155709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763233.283645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763233.283670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763233.283676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763233.283689:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2682 00000100:00080000:0.0:1789763233.283724:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763233.283733:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763233.283780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048968064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763233.283820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763233.283825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763233.283828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763233.283833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2682 00000100:00080000:0.0:1789763233.283837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763233.283841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763233.283853:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763233.283870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048968192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763233.283896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763233.283902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763233.283905:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763233.283910:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2682 00000100:00080000:0.0:1789763233.283914:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763233.283922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763233.283929:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048968064:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763233.283931:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763233.283954:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048968320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763233.283971:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763233.284013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763233.284015:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763233.284032:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2682 00000100:00080000:0.0:1789763233.284038:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763233.284044:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763233.284053:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763233.284074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048968448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763233.284086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763233.284105:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763233.284110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763233.284114:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763233.284120:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2682 00000100:00080000:0.0:1789763233.284126:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763233.284132:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763233.284140:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763233.284155:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048968576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763233.284159:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763233/real 1789763233] req@ffff9811c2e2ad80 x1876700048968064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763233.284250:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048968064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763233.284263:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763233.284269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763233.284277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048968064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763233.284285:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048968192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763233.284320:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763233.284332:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763233/real 1789763233] req@ffff9811c2e28e00 x1876700048968192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763233.284342:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048968192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763233.284348:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763233.284366:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763233.284369:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048968192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763233.284399:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763233.284417:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763233.284426:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763233/real 1789763233] req@ffff9811c2e28700 x1876700048968320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763233.284435:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048968320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763233.284441:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763233.284444:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763233.284446:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763233.284450:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763233.284464:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763233.284470:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763233/real 1789763233] req@ffff9811c2e2bb80 x1876700048968448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763233.284509:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048968448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763233.284516:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763233.284518:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763233.284521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763233.284526:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763233.284551:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763233.284560:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763233/real 1789763233] req@ffff9811c2e2a680 x1876700048968576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763233.284569:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048968576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763233.284575:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763233.284578:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763233.284580:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763233.539628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763234.179678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763234.563599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763235.203731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763235.587588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763236.227621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763236.611612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763237.251731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763237.635595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763238.275653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763238.403703:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763238.403725:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763238.403733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763238.403749:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2687 00000100:00080000:1.0:1789763238.403767:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763238.403779:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763238.403861:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048968704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763238.403908:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763238.403913:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763238.403916:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763238.403921:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2687 00000100:00080000:1.0:1789763238.403927:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763238.403932:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763238.403972:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763238.403995:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048968832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763238.404015:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763238.404022:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763238.404029:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763238.404036:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048968704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763238.404038:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2687 00000100:00080000:1.0:1789763238.404045:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763238.404065:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763238.404073:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763238.404089:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048968960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763238.404102:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763238.404107:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763238.404110:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763238.404126:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2687 00000100:00080000:1.0:1789763238.404134:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763238.404140:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763238.404149:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763238.404165:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048969088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763238.404183:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763238.404189:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763238.404195:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763238.404198:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763238.404206:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2687 00000100:00080000:1.0:1789763238.404225:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763238.404232:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763238.404241:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763238.404257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763238/real 1789763238] req@ffff9811c92c5f80 x1876700048968704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763238.404260:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048969216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763238.404364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048968704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763238.404390:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763238.404411:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763238.404418:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048968704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763238.404453:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048968832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763238.404551:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763238.404593:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763238/real 1789763238] req@ffff9811c92c4a80 x1876700048968832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763238.404611:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048968832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763238.404623:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763238.404628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763238.404632:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048968832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763238.404641:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763238.404667:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763238.404679:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763238/real 1789763238] req@ffff9811c92c4380 x1876700048968960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763238.404693:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048968960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763238.404703:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763238.404706:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763238.404710:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048968960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763238.404717:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763238.404749:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763238.404769:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763238/real 1789763238] req@ffff9811c92c6680 x1876700048969088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763238.404794:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048969088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763238.404823:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763238.404832:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763238.404838:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763238.404848:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763238.404882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763238.404898:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763238/real 1789763238] req@ffff9811c92c5c00 x1876700048969216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763238.404913:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048969216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763238.404923:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763238.404927:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763238.404931:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763238.659646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763239.299644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763239.683558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763240.323678:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763240.707577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763241.347686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763241.731637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763242.371637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763242.755618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763243.395727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763243.523570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763243.523582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763243.523588:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763243.523601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2693 00000100:00080000:1.0:1789763243.523615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763243.523624:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763243.523696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048969344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763243.523733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763243.523739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763243.523741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763243.523747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2693 00000100:00080000:1.0:1789763243.523752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763243.523757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763243.523768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763243.523783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048969472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763243.523797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763243.523802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763243.523805:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763243.523810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2693 00000100:00080000:1.0:1789763243.523814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763243.523820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763243.523826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763243.523838:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048969600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763243.523852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763243.523857:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048969344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763243.523858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763243.523866:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763243.523883:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2693 00000100:00080000:1.0:1789763243.523889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763243.523894:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763243.523935:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763243.523957:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048969728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763243.523972:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763243.523976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763243.523979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763243.523984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2693 00000100:00080000:1.0:1789763243.523990:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763243.523999:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763243.524003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763243.524013:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763243.524030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048969856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763243.524073:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763243/real 1789763243] req@ffff9811c92c7480 x1876700048969344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763243.524111:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048969344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763243.524132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763243.524156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763243.524165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048969344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763243.524181:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048969472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763243.524222:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763243.524241:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763243/real 1789763243] req@ffff9811c92c4e00 x1876700048969472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763243.524258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048969472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763243.524269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763243.524272:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763243.524277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048969472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763243.524308:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763243.524343:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763243.524366:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763243/real 1789763243] req@ffff9811c92c4000 x1876700048969600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763243.524391:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048969600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763243.524429:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763243.524434:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763243.524439:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763243.524448:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763243.524532:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763243.524549:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763243/real 1789763243] req@ffff9811c92c5500 x1876700048969728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763243.524565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048969728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763243.524575:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763243.524579:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763243.524584:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763243.524591:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763243.524612:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763243.524623:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763243/real 1789763243] req@ffff9811c92c5180 x1876700048969856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763243.524637:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048969856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763243.524646:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763243.524649:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763243.524653:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048969856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763243.779637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763244.419655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763244.803602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763245.443779:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763245.827586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763246.467641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763246.851631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763247.491690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763247.875595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763248.515597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763248.643718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763248.643729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763248.643734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763248.643746:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2698 00000100:00080000:0.0:1789763248.643760:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763248.643769:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763248.643820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048969984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763248.643906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763248.643912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763248.643914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763248.643920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2698 00000100:00080000:0.0:1789763248.643926:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763248.643931:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763248.643944:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763248.643964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048970112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763248.643981:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763248.643986:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048969984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763248.643988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763248.643994:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763248.644007:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2698 00000100:00080000:0.0:1789763248.644012:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763248.644016:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763248.644022:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763248.644037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048970240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763248.644049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763248.644053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763248.644055:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763248.644060:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2698 00000100:00080000:0.0:1789763248.644076:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763248.644083:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763248.644091:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763248.644100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763248.644113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048970368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763248.644133:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763248.644138:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763248.644140:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763248.644146:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2698 00000100:00080000:0.0:1789763248.644153:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763248.644157:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763248.644163:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763248.644181:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048970496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763248.644193:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763248/real 1789763248] req@ffff9811c2e29180 x1876700048969984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763248.644225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048969984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763248.644239:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763248.644244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763248.644250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048969984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763248.644278:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048970112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763248.644310:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763248.644326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763248/real 1789763248] req@ffff9811c2e29880 x1876700048970112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763248.644342:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048970112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763248.644353:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763248.644357:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763248.644361:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048970112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763248.644368:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763248.644388:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763248.644398:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763248/real 1789763248] req@ffff9811c2e29c00 x1876700048970240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763248.644412:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048970240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763248.644441:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763248.644445:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763248.644450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763248.644457:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763248.644525:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763248.644540:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763248/real 1789763248] req@ffff9811c2e2b100 x1876700048970368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763248.644556:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048970368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763248.644566:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763248.644570:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763248.644574:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763248.644580:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763248.644600:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763248.644611:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763248/real 1789763248] req@ffff9811c2e2aa00 x1876700048970496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763248.644624:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048970496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763248.644634:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763248.644637:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763248.644650:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763248.899598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763249.539677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763249.923610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763250.563642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763250.947567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763251.587704:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763251.971565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763252.611686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763252.995647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763253.635711:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763253.763650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763253.763666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763253.763673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763253.763690:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2703 00000100:00080000:1.0:1789763253.763707:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763253.763741:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763253.763801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048970624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763253.763870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763253.763879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763253.763883:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763253.763894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2703 00000100:00080000:1.0:1789763253.763907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763253.763918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763253.763939:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763253.763957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048970624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763253.763984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048970752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763253.764018:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763253.764027:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763253.764031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763253.764040:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2703 00000100:00080000:1.0:1789763253.764051:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763253.764064:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763253.764073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763253.764084:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763253.764110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048970880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763253.764127:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763253.764137:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789763253.764139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763253/real 1789763253] req@ffff9811c92c7b80 x1876700048970624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763253.764154:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763253.764182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2703 00000100:00080000:1.0:1789763253.764191:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763253.764193:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048970624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763253.764212:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789763253.764227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763253.764248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789763253.764252:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763253.764260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048970624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763253.764285:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048971008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763253.764304:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763253.764314:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048970752:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763253.764315:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763253.764326:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763253.764335:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2703 00000100:00080000:1.0:1789763253.764339:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763253.764344:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763253.764357:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763253.764378:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048971136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763253.764420:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763253.764446:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763253/real 1789763253] req@ffff9811c92c5880 x1876700048970752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763253.764513:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048970752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763253.764526:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763253.764531:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763253.764536:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048970752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763253.764545:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763253.764619:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763253.764636:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763253/real 1789763253] req@ffff9811c92c6d80 x1876700048970880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763253.764651:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048970880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763253.764662:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763253.764666:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763253.764670:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048970880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763253.764693:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763253.764717:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763253.764729:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763253/real 1789763253] req@ffff9811c92c5180 x1876700048971008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763253.764743:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048971008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763253.764753:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763253.764757:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763253.764760:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763253.764768:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763253.764787:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763253.764798:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763253/real 1789763253] req@ffff9811c92c5500 x1876700048971136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763253.764867:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048971136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763253.764879:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763253.764883:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763253.764887:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763254.019583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763254.659666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763255.043666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763255.683690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763256.067614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763256.707766:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763257.091633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763257.731754:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763258.115606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763258.755766:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763258.883698:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763258.883717:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763258.883725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763258.883745:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2708 00000100:00080000:2.0:1789763258.883766:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763258.883782:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763258.883890:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048971264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763258.883941:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763258.883947:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763258.883949:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763258.883955:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2708 00000100:00080000:2.0:1789763258.883961:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763258.883967:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763258.883982:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763258.884004:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048971392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763258.884031:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048971264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763258.884076:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763258.884084:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763258.884087:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763258.884111:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2708 00000100:00080000:2.0:1789763258.884115:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763258.884120:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763258.884126:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763258.884138:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048971520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763258.884146:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763258.884149:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763258.884151:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763258.884154:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2708 00000100:00080000:2.0:1789763258.884157:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763258.884159:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763258.884163:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763258.884174:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048971648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763258.884182:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763258.884185:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763258.884186:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763258.884188:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2708 00000100:00080000:2.0:1789763258.884199:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763258.884203:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763258.884208:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763258.884217:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048971776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763258.884230:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048971392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763258.884281:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763258.884313:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763258.884375:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763259.139580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763259.779678:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763260.163600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763260.803752:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763261.187598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763261.827734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763261.955836:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763261.955891:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763261.955912:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763261.955967:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763261.955990:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763261.956013:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789763261.956022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763258/real 1789763261] req@ffff9811d8639500 x1876700048971392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789763261.956043:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:1.0:1789763261.956093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048971392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763261.956118:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763261.956127:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763261.956135:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048971392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763261.956150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763258/real 1789763261] req@ffff9811d8639880 x1876700048971520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763261.956176:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048971520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763261.956192:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763261.956198:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763261.956205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763261.956219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763258/real 1789763261] req@ffff9811d863b100 x1876700048971648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763261.956243:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048971648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763261.956260:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763261.956287:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763261.956295:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763261.956310:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763258/real 1789763261] req@ffff9811d8639f80 x1876700048971264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763261.956337:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048971264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763261.956353:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763261.956359:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763261.956366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048971264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763261.956411:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763258/real 1789763261] req@ffff9811d8639c00 x1876700048971776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763261.956437:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048971776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763261.956451:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763261.956458:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763261.956501:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048971776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763262.211587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763262.851823:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763263.235556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763263.875730:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763264.003615:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763264.003630:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763264.003637:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763264.003653:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2713 00000100:00080000:0.0:1789763264.003670:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763264.003682:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763264.003775:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048971904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763264.003851:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763264.003857:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763264.003860:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763264.003866:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2713 00000100:00080000:0.0:1789763264.003872:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763264.003878:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763264.003894:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763264.003916:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048972032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763264.003934:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763264.003942:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763264.003947:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048971904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763264.003948:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763264.003963:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2713 00000100:00080000:0.0:1789763264.003971:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763264.003978:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763264.003989:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763264.004014:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048972160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763264.004035:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763264.004041:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763264.004045:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763264.004053:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2713 00000800:00000100:1.0:1789763264.004071:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763264.004083:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763264.004099:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763264.004112:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763264.004153:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763264/real 1789763264] req@ffff9811c2e28380 x1876700048971904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763264.004158:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048972288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763264.004218:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763264.004226:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763264.004231:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763264.004242:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2713 00000100:00080000:0.0:1789763264.004250:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763264.004259:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763264.004266:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048971904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763264.004270:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763264.004299:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048972416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763264.004318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763264.004334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763264.004345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048971904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763264.004399:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048972032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763264.004461:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763264.004605:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763264/real 1789763264] req@ffff9811c2e28a80 x1876700048972032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763264.004636:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048972032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763264.004656:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763264.004664:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763264.004676:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048972032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763264.004695:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763264.004777:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763264.004805:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763264/real 1789763264] req@ffff9811c2e29f80 x1876700048972160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763264.004832:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048972160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763264.004847:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763264.004853:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763264.004859:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763264.004887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763264.004936:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763264.004962:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763264/real 1789763264] req@ffff9811c2e2b800 x1876700048972288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763264.004986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048972288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763264.005003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763264.005010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763264.005017:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763264.005032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763264.005073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763264.005101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763264/real 1789763264] req@ffff9811c2e2a300 x1876700048972416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763264.005129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048972416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763264.005147:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763264.005154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763264.005161:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763264.259566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763264.899772:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763265.283573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763265.923822:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763266.307577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763266.947702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763267.331556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763267.971689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763268.355553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763268.995799:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763269.123621:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763269.123636:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763269.123644:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763269.123661:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2718 00000100:00080000:0.0:1789763269.123678:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763269.123690:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763269.123778:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048972544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763269.123818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763269.123847:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763269.123851:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763269.123858:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2718 00000100:00080000:0.0:1789763269.123865:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763269.123877:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763269.123881:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048972544:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763269.123895:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763269.123919:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048972672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763269.123937:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763269.123943:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763269.123946:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763269.123952:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2718 00000100:00080000:0.0:1789763269.123977:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763269.123982:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763269.123990:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763269.123998:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763269.124013:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048972800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763269.124029:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763269.124046:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763269.124049:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763269.124055:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2718 00000100:00080000:0.0:1789763269.124062:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763269.124071:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763269.124073:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763269/real 1789763269] req@ffff9811c2e2b480 x1876700048972544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763269.124090:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763269.124117:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048972928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763269.124132:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048972544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763269.124135:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763269.124151:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763269.124157:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763269.124172:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2718 00000100:00080000:1.0:1789763269.124174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763269.124179:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763269.124187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763269.124194:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763269.124201:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763269.124215:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048973056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763269.124221:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048972544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763269.124269:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048972672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763269.124317:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763269.124336:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763269/real 1789763269] req@ffff9811c2e29500 x1876700048972672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763269.124364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048972672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763269.124376:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763269.124380:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763269.124385:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048972672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763269.124393:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763269.124415:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763269.124428:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763269/real 1789763269] req@ffff9811c2e28000 x1876700048972800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763269.124442:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048972800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763269.124512:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763269.124518:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763269.124523:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763269.124532:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763269.124569:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763269.124584:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763269/real 1789763269] req@ffff9811c2e2a300 x1876700048972928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763269.124599:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048972928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763269.124608:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763269.124612:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763269.124615:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048972928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763269.124622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763269.124641:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763269.124652:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763269/real 1789763269] req@ffff9811c2e2b800 x1876700048973056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763269.124667:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048973056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763269.124676:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763269.124680:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763269.124693:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763269.379551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763270.019784:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763270.403539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763271.043789:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763271.427581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763272.067777:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763272.451567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763273.091742:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763273.475625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763274.115688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763274.243634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763274.243645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763274.243650:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763274.243663:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2723 00000100:00080000:1.0:1789763274.243694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763274.243703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763274.243750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048973184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763274.243788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763274.243793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763274.243795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763274.243800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2723 00000100:00080000:1.0:1789763274.243804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763274.243808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763274.243818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763274.243833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048973312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763274.243847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763274.243851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763274.243853:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763274.243857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2723 00000100:00080000:1.0:1789763274.243860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763274.243871:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048973184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763274.243888:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763274.243896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763274.243911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048973440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763274.243924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763274.243928:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763274.243930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763274.243934:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2723 00000100:00080000:1.0:1789763274.243942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763274.243947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763274.243951:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763274.243964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048973568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763274.243974:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763274.243981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1789763274.243984:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763274.243985:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763274.244006:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2723 00000100:00080000:1.0:1789763274.244010:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763274.244014:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763274.244019:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763274.244031:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048973696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763274.244088:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763274/real 1789763274] req@ffff9811c92c4000 x1876700048973184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763274.244122:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048973184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763274.244141:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763274.244148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763274.244156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048973184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763274.244169:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048973312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763274.244222:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763274.244263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763274/real 1789763274] req@ffff9811c92c4e00 x1876700048973312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763274.244288:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048973312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763274.244304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763274.244309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763274.244330:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048973312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763274.244369:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763274.244418:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763274.244520:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763274/real 1789763274] req@ffff9811c92c7480 x1876700048973440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763274.244549:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048973440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763274.244567:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763274.244573:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763274.244581:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763274.244595:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763274.244655:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763274.244705:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763274/real 1789763274] req@ffff9811c92c5c00 x1876700048973568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763274.244729:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048973568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763274.244745:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763274.244751:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763274.244758:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763274.244770:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763274.244809:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763274.244844:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763274/real 1789763274] req@ffff9811c92c6680 x1876700048973696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763274.244870:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048973696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763274.244886:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763274.244891:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763274.244898:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048973696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763274.499619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763275.140712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763275.523563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763276.163832:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763276.547583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763277.187681:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763277.571558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763278.211687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763278.595623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763279.235701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763279.363594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763279.363607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763279.363613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763279.363627:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2728 00000100:00080000:1.0:1789763279.363661:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763279.363690:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763279.363754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048973824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763279.363784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763279.363788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763279.363790:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763279.363794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2728 00000100:00080000:1.0:1789763279.363798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763279.363803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763279.363815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763279.363824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048973824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763279.363830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048973952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763279.363843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763279.363846:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763279.363848:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763279.363852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2728 00000100:00080000:1.0:1789763279.363855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763279.363858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763279.363899:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763279.363909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048974080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763279.363917:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763279.363920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763279.363922:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763279.363924:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2728 00000100:00080000:1.0:1789763279.363928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763279.363931:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763279.363934:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763279.363940:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763279.363945:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048974208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763279.363968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763279.363971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763279.363973:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763279.363976:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2728 00000100:00080000:1.0:1789763279.363979:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763279.363982:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763279.363985:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763279.363991:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048974336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763279.364019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763279/real 1789763279] req@ffff9811c92c4380 x1876700048973824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763279.364058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048973824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763279.364068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763279.364072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763279.364076:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048973824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763279.364124:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048973952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763279.364151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763279.364182:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763279/real 1789763279] req@ffff9811c92c4a80 x1876700048973952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763279.364195:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048973952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763279.364240:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763279.364244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763279.364247:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048973952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763279.364252:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763279.364268:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763279.364276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763279/real 1789763279] req@ffff9811c92c5f80 x1876700048974080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763279.364284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048974080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763279.364291:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763279.364294:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763279.364296:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763279.364300:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763279.364312:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763279.364322:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763279/real 1789763279] req@ffff9811c92c6a00 x1876700048974208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763279.364351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048974208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763279.364369:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763279.364374:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763279.364387:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763279.364393:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763279.364411:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763279.364437:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763279/real 1789763279] req@ffff9811c92c6300 x1876700048974336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763279.364447:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048974336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763279.364453:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763279.364455:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763279.364458:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763279.619580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763280.259751:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763280.643615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763281.283721:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763281.667551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763282.307590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763282.691597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763283.331636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763283.715570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763284.355662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763284.483567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763284.483577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763284.483581:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763284.483593:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2734 00000100:00080000:0.0:1789763284.483616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763284.483625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763284.483680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048974464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763284.483711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763284.483727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763284.483742:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763284.483747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2734 00000100:00080000:0.0:1789763284.483751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763284.483755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763284.483765:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048974464:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763284.483766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763284.483782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048974592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763284.483802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763284.483805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763284.483807:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763284.483809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2734 00000100:00080000:0.0:1789763284.483812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763284.483814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763284.483817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763284.483825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048974720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763284.483831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763284.483834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763284.483835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763284.483838:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2734 00000100:00080000:0.0:1789763284.483848:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763284.483852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763284.483855:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763284.483861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763284.483877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048974848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763284.483884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763284.483887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763284.483888:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763284.483890:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2734 00000100:00080000:0.0:1789763284.483893:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763284.483897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763284.483901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763284.483905:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763284/real 1789763284] req@ffff9811c2e29f80 x1876700048974464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763284.483912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048974976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763284.483931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048974464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763284.483941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763284.483945:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763284.483949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048974464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763284.483975:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048974592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763284.484001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763284.484034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763284/real 1789763284] req@ffff9811c2e28a80 x1876700048974592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763284.484044:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048974592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763284.484050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763284.484053:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763284.484055:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048974592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763284.484059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763284.484081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763284.484098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763284/real 1789763284] req@ffff9811c2e28380 x1876700048974720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763284.484112:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048974720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763284.484121:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763284.484124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763284.484128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763284.484135:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763284.484164:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763284.484172:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763284/real 1789763284] req@ffff9811c2e2aa00 x1876700048974848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763284.484184:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048974848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763284.484200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763284.484204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763284.484208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763284.484214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763284.484255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763284.484264:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763284/real 1789763284] req@ffff9811c2e2b100 x1876700048974976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763284.484272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048974976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763284.484278:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763284.484280:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763284.484282:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048974976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763284.739588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763285.379730:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763285.763566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763286.403713:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763286.787614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763287.427735:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763287.811543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763288.451747:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763288.835526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763289.475661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763289.603647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763289.603658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763289.603663:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763289.603675:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2739 00000100:00080000:0.0:1789763289.603687:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763289.603696:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763289.603741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048975104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763289.603764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763289.603768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763289.603788:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763289.603793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2739 00000100:00080000:0.0:1789763289.603796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763289.603802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763289.603804:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048975104:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763289.603812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763289.603824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048975232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763289.603841:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763289.603843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763289.603845:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763289.603848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2739 00000100:00080000:0.0:1789763289.603852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763289.603854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763289.603857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763289.603865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048975360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763289.603871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763289.603874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763289.603876:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763289.603880:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763289.603884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2739 00000100:00080000:0.0:1789763289.603888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763289.603890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763289.603894:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763289.603908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048975488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763289.603914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763289.603917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763289.603918:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763289.603925:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2739 00000100:00080000:0.0:1789763289.603928:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763289.603929:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763289/real 1789763289] req@ffff9811c2e29c00 x1876700048975104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763289.603934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763289.603942:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763289.603953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048975616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763289.603957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048975104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763289.603988:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763289.603993:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763289.603999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048975104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763289.604029:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048975232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763289.604059:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763289.604069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763289/real 1789763289] req@ffff9811c2e29880 x1876700048975232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763289.604082:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048975232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763289.604091:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763289.604095:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763289.604098:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048975232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763289.604105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763289.604127:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763289.604137:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763289/real 1789763289] req@ffff9811c2e29180 x1876700048975360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763289.604145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048975360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763289.604151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763289.604153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763289.604163:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763289.604173:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763289.604190:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763289.604197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763289/real 1789763289] req@ffff9811c2e2a680 x1876700048975488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763289.604206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048975488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763289.604211:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763289.604214:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763289.604216:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763289.604220:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763289.604238:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763289.604248:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763289/real 1789763289] req@ffff9811c2e2bb80 x1876700048975616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763289.604261:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048975616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763289.604270:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763289.604274:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763289.604277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048975616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763289.859519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763290.499588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763290.883541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763291.523620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763291.907601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763292.547721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763292.931523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763293.571668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763293.955657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763294.595663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763294.723609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763294.723617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763294.723621:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763294.723629:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2744 00000100:00080000:1.0:1789763294.723638:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763294.723644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763294.723677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048975744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763294.723699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763294.723702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763294.723703:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763294.723706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2744 00000100:00080000:1.0:1789763294.723708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763294.723711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763294.723718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763294.723729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048975872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763294.723738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763294.723740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763294.723760:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763294.723761:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048975744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763294.723773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2744 00000100:00080000:1.0:1789763294.723779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763294.723783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763294.723786:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763294.723801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048976000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763294.723809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763294.723813:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763294.723814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763294.723817:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2744 00000100:00080000:1.0:1789763294.723820:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763294.723822:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763294.723826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763294.723834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048976128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763294.723840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763294.723843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763294.723844:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763294.723846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2744 00000100:00080000:1.0:1789763294.723850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763294.723853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789763294.723860:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763294.723864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763294.723882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048976256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763294.723918:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763294/real 1789763294] req@ffff9811c92c4700 x1876700048975744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763294.723944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048975744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763294.723958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763294.723961:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763294.723965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048975744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763294.723972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048975872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763294.723993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763294.724003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763294/real 1789763294] req@ffff9811c92c7800 x1876700048975872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763294.724017:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048975872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763294.724026:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763294.724030:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763294.724034:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048975872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763294.724065:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763294.724083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763294.724091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763294/real 1789763294] req@ffff9811c92c7100 x1876700048976000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763294.724113:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048976000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763294.724119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763294.724121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763294.724123:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763294.724127:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763294.724141:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763294.724148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763294/real 1789763294] req@ffff9811c92c6300 x1876700048976128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763294.724156:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048976128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763294.724161:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763294.724163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763294.724165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763294.724168:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763294.724178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763294.724184:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763294/real 1789763294] req@ffff9811c92c6a00 x1876700048976256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763294.724191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048976256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763294.724219:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763294.724221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763294.724224:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763294.979564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763295.619602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763296.003550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763296.643621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763297.027554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763297.667590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763298.051569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763298.691619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763299.075543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763299.715617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763299.843586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763299.843597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763299.843601:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763299.843611:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2749 00000100:00080000:1.0:1789763299.843620:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763299.843627:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763299.843661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048976384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763299.843689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763299.843692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763299.843695:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763299.843702:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2749 00000100:00080000:1.0:1789763299.843705:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763299.843709:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763299.843718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763299.843728:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048976384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763299.843734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048976512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763299.843745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763299.843749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763299.843750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763299.843761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2749 00000100:00080000:1.0:1789763299.843781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763299.843789:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763299.843794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763299.843801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763299.843816:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048976640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763299.843829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763299.843832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763299.843835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763299.843839:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763299/real 1789763299] req@ffff9811c92c5f80 x1876700048976384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763299.843844:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2749 00000100:00080000:1.0:1789763299.843851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763299.843856:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763299.843862:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763299.843866:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048976384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763299.843875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048976768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763299.843886:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763299.843887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763299.843895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763299.843899:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789763299.843899:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763299.843910:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048976384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763299.843910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2749 00000100:00080000:1.0:1789763299.843917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763299.843924:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763299.843929:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763299.843942:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048976512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763299.843944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048976896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763299.843976:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763299.843990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763299/real 1789763299] req@ffff9811c92c4a80 x1876700048976512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763299.844005:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048976512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763299.844014:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763299.844019:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763299.844022:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048976512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763299.844030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763299.844053:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763299.844064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763299/real 1789763299] req@ffff9811c92c4380 x1876700048976640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763299.844073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048976640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763299.844079:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763299.844093:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763299.844096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763299.844100:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763299.844114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763299.844121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763299/real 1789763299] req@ffff9811c92c6680 x1876700048976768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763299.844141:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048976768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763299.844146:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763299.844148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763299.844151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763299.844165:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763299.844179:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763299.844188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763299/real 1789763299] req@ffff9811c92c5c00 x1876700048976896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763299.844196:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048976896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763299.844202:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763299.844204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763299.844207:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048976896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763300.099600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763300.739599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763301.123536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763301.763629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763302.147557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763302.787616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763303.171531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763303.811621:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763304.195544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763304.835565:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763304.963541:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763304.963552:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763304.963556:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763304.963567:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2754 00000100:00080000:1.0:1789763304.963577:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763304.963584:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763304.963618:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048977024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763304.963643:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763304.963647:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763304.963649:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763304.963653:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2754 00000100:00080000:1.0:1789763304.963656:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763304.963661:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763304.963670:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763304.963687:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048977024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763304.963714:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048977152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763304.963727:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763304.963731:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763304.963733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763304.963738:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2754 00000100:00080000:1.0:1789763304.963742:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763304.963746:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763304.963760:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763304.963766:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763304.963780:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048977280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763304.963785:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763304/real 1789763304] req@ffff9811c92c7480 x1876700048977024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763304.963802:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763304.963808:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789763304.963812:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048977024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763304.963813:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763304.963818:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2754 00000100:00080000:1.0:1789763304.963824:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763304.963832:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763304.963836:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789763304.963838:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763304.963842:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763304.963847:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048977024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763304.963854:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048977408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763304.963866:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763304.963870:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048977152:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763304.963872:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763304.963877:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763304.963884:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2754 00000100:00080000:1.0:1789763304.963889:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763304.963893:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763304.963898:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763304.963900:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763304.963912:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048977536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763304.963920:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763304/real 1789763304] req@ffff9811c92c4e00 x1876700048977152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763304.963940:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048977152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763304.963949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763304.963953:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763304.963957:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048977152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763304.963961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763304.963977:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763304.963985:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763304/real 1789763304] req@ffff9811c92c4000 x1876700048977280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763304.963993:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048977280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763304.963998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763304.964000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763304.964002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763304.964021:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763304.964040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763304.964047:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763304/real 1789763304] req@ffff9811c92c5500 x1876700048977408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763304.964055:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048977408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763304.964060:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763304.964062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763304.964064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763304.964076:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763304.964086:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763304.964092:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763304/real 1789763304] req@ffff9811c92c5180 x1876700048977536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763304.964099:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048977536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763304.964103:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763304.964105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763304.964113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763305.219519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763305.859637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763306.243539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763306.883589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763307.267523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763307.907623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763308.291541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763308.931571:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763309.315533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763309.955636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763310.083555:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763310.083566:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763310.083570:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763310.083582:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2759 00000100:00080000:3.0:1789763310.083593:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763310.083601:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763310.083643:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048977664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763310.083668:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763310.083671:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763310.083673:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763310.083676:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2759 00000100:00080000:3.0:1789763310.083678:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763310.083681:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763310.083690:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763310.083699:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048977792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763310.083712:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763310.083716:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763310.083724:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048977664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763310.083741:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763310.083749:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2759 00000100:00080000:3.0:1789763310.083754:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763310.083759:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763310.083765:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763310.083779:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048977920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763310.083787:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763310.083790:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763310.083791:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763310.083794:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2759 00000100:00080000:3.0:1789763310.083798:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763310.083800:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763310.083803:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763310.083811:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048978048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763310.083817:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763310.083822:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763310.083823:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763310.083827:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763310.083832:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2759 00000100:00080000:3.0:1789763310.083837:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763310.083841:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763310.083847:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763310.083870:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048978176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763310.083875:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763310/real 1789763310] req@ffff9811e1a23100 x1876700048977664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763310.083902:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048977664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763310.083912:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763310.083915:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763310.083919:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048977664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763310.083940:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048977792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763310.083956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763310.083964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763310/real 1789763310] req@ffff9811e1a23800 x1876700048977792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763310.083972:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048977792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763310.083978:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763310.083980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763310.083982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048977792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763310.083985:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763310.083996:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763310.084002:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763310/real 1789763310] req@ffff9811e1a20a80 x1876700048977920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763310.084028:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048977920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763310.084034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763310.084036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763310.084038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048977920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763310.084042:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763310.084056:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763310.084063:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763310/real 1789763310] req@ffff9811e1a21f80 x1876700048978048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763310.084071:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048978048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763310.084076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763310.084078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763310.084080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763310.084083:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763310.084092:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763310.084097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763310/real 1789763310] req@ffff9811e1a20000 x1876700048978176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763310.084104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048978176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763310.084117:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763310.084119:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763310.084122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763310.339582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763310.979640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763311.363575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763312.003729:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763312.387648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763313.027765:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763313.411598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763314.051694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763314.435567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763315.075751:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763315.203583:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763315.203595:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763315.203601:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763315.203614:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2764 00000100:00080000:3.0:1789763315.203628:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763315.203638:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763315.203688:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700048978304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763315.203728:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763315.203733:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763315.203735:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763315.203740:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2764 00000100:00080000:3.0:1789763315.203745:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763315.203749:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763315.203761:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763315.203775:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700048978432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763315.203790:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763315.203795:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763315.203819:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763315.203825:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2764 00000100:00080000:3.0:1789763315.203831:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763315.203837:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763315.203845:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763315.203854:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048978304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763315.203868:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700048978560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763315.203885:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763315.203889:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763315.203891:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763315.203896:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2764 00000100:00080000:3.0:1789763315.203901:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763315.203905:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763315.203911:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763315.203925:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700048978688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763315.203937:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763315.203940:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763315.203942:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763315.203947:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2764 00000100:00080000:3.0:1789763315.203953:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763315.203957:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763315.203964:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763315.203968:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763315.203996:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700048978816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763315.204039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763315/real 1789763315] req@ffff9811e1a20e00 x1876700048978304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763315.204068:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700048978304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763315.204082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763315.204087:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763315.204093:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048978304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763315.204102:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048978432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763315.204130:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763315.204144:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763315/real 1789763315] req@ffff9811e1a20380 x1876700048978432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763315.204159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700048978432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763315.204169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763315.204186:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763315.204191:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048978432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763315.204221:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763315.204247:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763315.204261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763315/real 1789763315] req@ffff9811e1a20000 x1876700048978560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763315.204276:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700048978560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763315.204286:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763315.204290:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763315.204294:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763315.204301:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763315.204320:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763315.204330:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763315/real 1789763315] req@ffff9811e1a21f80 x1876700048978688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763315.204344:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700048978688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763315.204353:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763315.204356:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763315.204360:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763315.204366:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763315.204433:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763315.204449:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763315/real 1789763315] req@ffff9811e1a20a80 x1876700048978816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763315.204523:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700048978816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763315.204536:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763315.204539:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763315.204543:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048978816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763315.459609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763316.099704:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763316.483548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763317.123583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763317.507531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763318.147743:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763318.531570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763319.171727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763319.555601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763320.195743:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763320.323575:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763320.323588:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763320.323594:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763320.323607:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2769 00000100:00080000:3.0:1789763320.323620:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763320.323630:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763320.323691:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700048978944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763320.323727:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763320.323732:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763320.323734:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763320.323739:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2769 00000100:00080000:3.0:1789763320.323743:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763320.323747:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763320.323758:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763320.323775:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700048979072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763320.323790:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763320.323796:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763320.323798:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763320.323803:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2769 00000100:00100000:0.0:1789763320.323828:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048978944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763320.323831:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763320.323875:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763320.323888:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763320.323919:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700048979200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763320.323942:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763320.323949:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763320.323953:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763320.323964:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763320.323983:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2769 00000100:00080000:3.0:1789763320.323994:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763320.324003:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763320.324015:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763320.324034:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763320/real 1789763320] req@ffff9811e1a23800 x1876700048978944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763320.324047:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700048979328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763320.324075:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763320.324079:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700048978944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763320.324085:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763320.324094:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763320.324109:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2769 00000100:00080000:3.0:1789763320.324129:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763320.324136:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763320.324138:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763320.324155:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763320.324158:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763320.324169:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048978944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763320.324179:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700048979456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763320.324189:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048979072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763320.324240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763320.324276:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763320/real 1789763320] req@ffff9811e1a23100 x1876700048979072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763320.324294:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700048979072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763320.324305:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763320.324319:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763320.324324:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048979072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763320.324385:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763320.324416:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763320.324431:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763320/real 1789763320] req@ffff9811e1a20700 x1876700048979200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763320.324447:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700048979200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763320.324458:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763320.324500:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763320.324505:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763320.324513:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763320.324539:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763320.324553:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763320/real 1789763320] req@ffff9811e1a21500 x1876700048979328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763320.324567:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700048979328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763320.324588:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763320.324593:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763320.324597:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763320.324604:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763320.324630:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763320.324643:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763320/real 1789763320] req@ffff9811e1a23480 x1876700048979456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763320.324666:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700048979456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763320.324676:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763320.324680:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763320.324684:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763320.579568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763321.219798:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763321.603529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763322.243659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763322.627588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763323.267705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763323.651560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763324.291683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763324.675591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763325.315733:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763325.443662:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763325.443678:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763325.443685:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763325.443703:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2775 00000100:00080000:1.0:1789763325.443720:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763325.443731:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763325.443794:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048979584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763325.443842:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763325.443850:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763325.443854:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763325.443863:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2775 00000100:00080000:1.0:1789763325.443872:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763325.443880:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763325.443899:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763325.443929:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048979584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763325.443932:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048979712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763325.443966:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763325.443971:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763325.443974:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763325.443980:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2775 00000100:00080000:1.0:1789763325.443986:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763325.443992:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763325.444023:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763325.444041:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048979840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763325.444056:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763325.444060:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763325.444063:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763325.444068:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2775 00000100:00080000:1.0:1789763325.444074:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763325.444080:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763325.444086:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763325.444104:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048979968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763325.444121:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763325.444126:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763325.444129:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763325.444137:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048979712:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763325.444144:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2775 00000100:00080000:1.0:1789763325.444153:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763325.444158:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763325.444165:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763325.444193:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048980096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763325.444216:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763325.444270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763325.444372:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763325.699575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763326.341581:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763326.723550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763327.363741:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763327.747627:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763328.389446:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763328.516231:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763328.516355:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763328.516386:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763328.516568:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763328.516600:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789763328.516629:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763325/real 1789763328] req@ffff9811c92c5880 x1876700048979712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789763328.516640:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763328.516705:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789763328.516722:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048979712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763328.516749:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763328.516760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763328.516768:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048979712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763328.516783:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763325/real 1789763328] req@ffff9811c92c7b80 x1876700048979840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763328.516812:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048979840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763328.516830:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763328.516839:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763328.516848:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763328.516866:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763325/real 1789763328] req@ffff9811c92c5180 x1876700048979968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763328.516892:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048979968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763328.516925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763328.516932:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763328.516969:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048979968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763328.516984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763325/real 1789763328] req@ffff9811c92c6d80 x1876700048979584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763328.517015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048979584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763328.517031:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763328.517038:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763328.517045:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048979584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763328.517549:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763325/real 1789763328] req@ffff9811c92c5500 x1876700048980096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763328.517597:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048980096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763328.517616:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763328.517640:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763328.517649:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763328.771551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763329.411581:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763329.795579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763330.435689:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763330.563582:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763330.563594:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763330.563599:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763330.563623:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2780 00000100:00080000:1.0:1789763330.563636:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763330.563645:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763330.563688:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048980224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763330.563710:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763330.563713:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763330.563715:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763330.563718:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2780 00000100:00080000:1.0:1789763330.563720:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763330.563723:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763330.563732:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763330.563742:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048980352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763330.563752:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763330.563752:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048980224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763330.563760:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763330.563764:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763330.563779:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2780 00000100:00080000:1.0:1789763330.563784:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763330.563789:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763330.563815:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763330.563829:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048980480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763330.563842:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763330.563846:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763330.563850:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763330.563851:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763330.563856:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2780 00000100:00080000:1.0:1789763330.563863:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763330.563868:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763330.563874:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763330.563885:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048980608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763330.563892:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763330.563895:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763330.563897:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763330.563902:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2780 00000100:00080000:1.0:1789763330.563907:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763330.563912:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763330/real 1789763330] req@ffff9811c92c4000 x1876700048980224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763330.563915:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763330.563925:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763330.563943:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048980736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763330.563957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048980224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763330.563987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763330.563994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763330.563999:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048980224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763330.564105:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048980352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763330.564139:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763330.564156:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763330/real 1789763330] req@ffff9811c92c4e00 x1876700048980352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763330.564169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048980352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763330.564180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763330.564184:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763330.564187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048980352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763330.564195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763330.564220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763330.564233:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763330/real 1789763330] req@ffff9811c92c7480 x1876700048980480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763330.564247:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048980480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763330.564286:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763330.564291:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763330.564295:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763330.564303:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763330.564333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763330.564346:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763330/real 1789763330] req@ffff9811c92c5c00 x1876700048980608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763330.564359:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048980608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763330.564370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763330.564375:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763330.564378:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763330.564385:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763330.564409:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763330.564422:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763330/real 1789763330] req@ffff9811c92c6680 x1876700048980736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763330.564435:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048980736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763330.564445:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763330.564449:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763330.564490:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048980736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763330.819592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763331.459756:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763331.843607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763332.483694:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763332.867547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763333.507675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763333.891557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763334.531723:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763334.915541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763335.555666:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763335.683624:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763335.683662:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763335.683681:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763335.683702:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2785 00000100:00080000:0.0:1789763335.683719:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763335.683731:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763335.683787:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048980864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763335.683833:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763335.683839:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763335.683842:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763335.683849:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2785 00000100:00080000:0.0:1789763335.683854:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763335.683882:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763335.683900:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763335.683908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048980864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763335.683937:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048980992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763335.683967:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763335.683973:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763335.683977:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763335.683985:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2785 00000100:00080000:0.0:1789763335.684005:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763335.684015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763335.684016:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763335.684031:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763335.684052:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048981120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763335.684080:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763335.684085:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789763335.684091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763335/real 1789763335] req@ffff9811c2e28700 x1876700048980864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763335.684092:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763335.684105:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2785 00000100:00080000:0.0:1789763335.684117:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763335.684127:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763335.684138:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763335.684139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048980864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.684164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.684175:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763335.684178:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048981248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763335.684187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048980864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763335.684218:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763335.684252:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763335.684256:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763335.684262:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048980992:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763335.684282:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2785 00000100:00080000:0.0:1789763335.684295:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763335.684305:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763335.684317:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789763335.684322:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763335.684353:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048981376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763335.684376:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763335/real 1789763335] req@ffff9811c2e28e00 x1876700048980992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.684416:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048980992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.684434:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.684441:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763335.684449:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048980992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763335.684520:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763335.684566:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763335.684587:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763335/real 1789763335] req@ffff9811c2e2ad80 x1876700048981120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.684627:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048981120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.684639:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.684643:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763335.684648:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763335.684656:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763335.684684:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763335.684697:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763335/real 1789763335] req@ffff9811c2e2bb80 x1876700048981248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.684722:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048981248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.684731:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.684735:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763335.684739:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763335.684762:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763335.684785:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763335.684804:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763335/real 1789763335] req@ffff9811c2e2a680 x1876700048981376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.684828:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048981376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.684869:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.684877:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763335.684884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763335.939567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763336.579730:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763336.963571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763337.603680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763337.987547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763338.627750:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763339.011527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763339.651602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763340.035573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763340.675720:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763340.803619:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763340.803630:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763340.803636:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763340.803648:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2790 00000100:00080000:0.0:1789763340.803678:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763340.803687:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763340.803732:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048981504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763340.803770:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763340.803775:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763340.803778:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763340.803782:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2790 00000100:00080000:0.0:1789763340.803786:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763340.803791:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763340.803804:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763340.803820:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048981632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763340.803832:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048981504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763340.803843:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763340.803850:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763340.803852:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763340.803859:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2790 00000100:00080000:0.0:1789763340.803864:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763340.803868:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763340.803874:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763340.803891:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048981760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763340.803903:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763340.803916:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763340.803918:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763340.803924:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2790 00000100:00080000:0.0:1789763340.803930:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763340.803934:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763340.803941:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763340.803952:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763340.803969:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048981888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763340.803982:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763340.803986:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763340.803988:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763340.803994:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2790 00000100:00080000:0.0:1789763340.803999:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763340.804004:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763340.804010:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763340.804016:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763340/real 1789763340] req@ffff9811c2e29180 x1876700048981504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763340.804031:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048982016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763340.804057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048981504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763340.804076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763340.804082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763340.804088:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048981504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763340.804118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048981632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763340.804151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763340.804168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763340/real 1789763340] req@ffff9811c2e29880 x1876700048981632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763340.804211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048981632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763340.804243:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763340.804248:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763340.804253:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048981632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763340.804261:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763340.804291:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763340.804306:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763340/real 1789763340] req@ffff9811c2e29c00 x1876700048981760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763340.804321:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048981760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763340.804331:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763340.804335:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763340.804339:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763340.804346:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763340.804365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763340.804376:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763340/real 1789763340] req@ffff9811c2e2b100 x1876700048981888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763340.804390:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048981888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763340.804399:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763340.804402:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763340.804416:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048981888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763340.804424:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763340.804448:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763340.804491:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763340/real 1789763340] req@ffff9811c2e2aa00 x1876700048982016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763340.804509:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048982016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763340.804520:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763340.804523:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763340.804527:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763341.059639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763341.699734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763342.083600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763342.723592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763343.107536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763343.747610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763344.131559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763344.771646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763345.155585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763345.795669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763345.923688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763345.923703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763345.923709:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763345.923726:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2795 00000100:00080000:0.0:1789763345.923743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763345.923755:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763345.923819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048982144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763345.923915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763345.923922:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763345.923926:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763345.923933:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2795 00000100:00080000:0.0:1789763345.923941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763345.923948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763345.923966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763345.923992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048982272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763345.924005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048982144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763345.924019:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763345.924028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763345.924031:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763345.924039:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2795 00000100:00080000:0.0:1789763345.924045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763345.924053:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763345.924061:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763345.924083:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048982400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763345.924100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763345.924107:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1789763345.924111:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763345.924113:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763345.924124:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2795 00000100:00080000:0.0:1789763345.924178:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763345.924188:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763345.924199:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763345.924214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763345/real 1789763345] req@ffff9811c2e28380 x1876700048982144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763345.924226:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048982528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763345.924247:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763345.924248:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048982144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763345.924262:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763345.924271:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763345.924277:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763345.924284:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2795 00000100:00080000:1.0:1789763345.924291:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763345.924295:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763345.924300:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048982144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763345.924308:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763345.924322:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763345.924346:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048982656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763345.924359:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048982272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763345.924409:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763345.924434:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763345/real 1789763345] req@ffff9811c2e28a80 x1876700048982272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763345.924458:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048982272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763345.924513:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763345.924518:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763345.924522:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048982272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763345.924530:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763345.924558:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763345.924572:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763345/real 1789763345] req@ffff9811c2e29f80 x1876700048982400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763345.924585:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048982400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763345.924594:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763345.924597:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763345.924601:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763345.924606:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763345.924624:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763345.924633:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763345/real 1789763345] req@ffff9811c2e2b800 x1876700048982528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763345.924656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048982528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763345.924665:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763345.924669:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763345.924672:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763345.924692:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763345.924713:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763345.924723:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763345/real 1789763345] req@ffff9811c2e2a300 x1876700048982656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763345.924736:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048982656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763345.924755:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763345.924757:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763345.924760:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048982656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763346.179599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763346.819754:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763347.203565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763347.843675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763348.227539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763348.867650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763349.251574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763349.891756:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763350.275580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763350.915724:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763351.043627:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763351.043644:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763351.043652:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763351.043686:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2800 00000100:00080000:1.0:1789763351.043703:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763351.043715:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763351.043770:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048982784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763351.043821:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763351.043826:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763351.043829:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763351.043835:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2800 00000100:00080000:1.0:1789763351.043842:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763351.043849:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763351.043862:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763351.043889:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048982912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763351.043891:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048982784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763351.043921:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763351.043927:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763351.043930:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763351.043935:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2800 00000100:00080000:1.0:1789763351.043939:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763351.043944:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763351.043950:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763351.043970:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048983040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763351.043996:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763351.044025:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763351.044032:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763351.044036:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763351.044046:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2800 00000100:00080000:1.0:1789763351.044056:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763351.044065:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763351.044073:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763351/real 1789763351] req@ffff9811c92c4380 x1876700048982784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789763351.044080:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763351.044134:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048982784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763351.044143:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048983168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763351.044183:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763351.044187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763351.044189:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763351.044194:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763351.044198:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2800 00000100:00080000:1.0:1789763351.044203:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763351.044213:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789763351.044221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:1.0:1789763351.044222:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763351.044239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048982784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763351.044248:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048983296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763351.044309:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048982912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763351.044373:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763351.044402:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763351/real 1789763351] req@ffff9811c92c4a80 x1876700048982912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763351.044431:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048982912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763351.044444:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763351.044448:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763351.044453:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048982912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763351.044505:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763351.044561:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763351.044584:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763351/real 1789763351] req@ffff9811c92c5f80 x1876700048983040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763351.044601:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048983040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763351.044612:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763351.044616:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763351.044621:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763351.044629:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763351.044671:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763351.044686:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763351/real 1789763351] req@ffff9811c92c6a00 x1876700048983168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763351.044711:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048983168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763351.044729:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763351.044736:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763351.044744:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763351.044757:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763351.044799:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763351.044823:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763351/real 1789763351] req@ffff9811c92c6300 x1876700048983296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763351.044847:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048983296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763351.044857:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763351.044861:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763351.044865:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763351.299545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763351.939597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763352.323599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763352.963811:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763353.347633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763353.987642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763354.371585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763355.011663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763355.395534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763356.035600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763356.163544:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763356.163553:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763356.163556:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763356.163564:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2805 00000100:00080000:0.0:1789763356.163573:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763356.163578:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763356.163610:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048983424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763356.163634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763356.163655:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763356.163658:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763356.163664:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2805 00000100:00080000:0.0:1789763356.163670:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763356.163675:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763356.163685:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763356.163703:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048983552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763356.163703:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048983424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763356.163729:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763356.163731:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763356.163733:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763356.163736:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2805 00000100:00080000:0.0:1789763356.163739:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763356.163743:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763356.163748:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763356.163759:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048983680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763356.163766:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763356.163769:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763356.163771:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763356.163773:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2805 00000100:00080000:0.0:1789763356.163780:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763356.163783:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763356.163794:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763356.163804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048983808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763356.163812:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763356.163818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763356.163820:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763356.163821:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763356.163824:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2805 00000100:00080000:0.0:1789763356.163839:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763356.163843:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763356.163849:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763356.163858:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048983936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763356.163887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763356/real 1789763356] req@ffff9811c2e28000 x1876700048983424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763356.163908:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048983424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763356.163918:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763356.163922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763356.163925:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048983424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763356.163954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048983552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763356.163982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763356.163997:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763356/real 1789763356] req@ffff9811c2e29500 x1876700048983552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763356.164012:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048983552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763356.164021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763356.164025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763356.164027:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048983552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763356.164033:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763356.164050:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763356.164059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763356/real 1789763356] req@ffff9811c2e2b480 x1876700048983680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763356.164067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048983680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763356.164099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763356.164102:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763356.164104:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763356.164109:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763356.164128:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763356.164136:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763356/real 1789763356] req@ffff9811c2e2a300 x1876700048983808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763356.164144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048983808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763356.164150:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763356.164152:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763356.164154:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763356.164158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763356.164170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763356.164175:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763356/real 1789763356] req@ffff9811c2e2b800 x1876700048983936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763356.164182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048983936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763356.164187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763356.164188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763356.164196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048983936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763356.419573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763357.059798:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763357.443615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763358.083712:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763358.467604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763359.107717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763359.491589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763360.131720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763360.515541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763361.155645:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763361.283591:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763361.283603:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763361.283609:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763361.283625:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2810 00000100:00080000:1.0:1789763361.283641:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763361.283652:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763361.283725:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048984064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763361.283760:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763361.283766:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763361.283769:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763361.283776:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2810 00000100:00080000:1.0:1789763361.283782:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763361.283789:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763361.283806:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763361.283837:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048984192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763361.283838:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048984064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763361.283857:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763361.283862:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763361.283865:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763361.283870:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2810 00000100:00080000:1.0:1789763361.283875:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763361.283881:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763361.283935:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763361.283955:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048984320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763361.283965:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763361.283974:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763361.283980:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763361.283983:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763361.283989:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2810 00000100:00080000:1.0:1789763361.283996:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763361.284003:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763361.284012:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763361.284072:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048984448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763361.284087:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763361/real 1789763361] req@ffff9811c92c7100 x1876700048984064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763361.284101:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763361.284106:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763361.284109:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763361.284115:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2810 00000100:00100000:2.0:1789763361.284119:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048984064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763361.284120:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763361.284127:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763361.284134:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763361.284137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763361.284143:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763361.284150:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048984064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763361.284151:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048984576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763361.284189:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048984192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763361.284229:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763361.284246:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763361/real 1789763361] req@ffff9811c92c7800 x1876700048984192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763361.284261:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048984192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763361.284271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763361.284274:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763361.284278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048984192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763361.284284:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763361.284304:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763361.284326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763361/real 1789763361] req@ffff9811c92c4700 x1876700048984320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763361.284339:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048984320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763361.284348:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763361.284351:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763361.284355:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763361.284360:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763361.284380:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763361.284389:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763361/real 1789763361] req@ffff9811c92c6300 x1876700048984448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763361.284401:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048984448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763361.284408:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763361.284411:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763361.284414:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763361.284419:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763361.284443:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763361.284501:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763361/real 1789763361] req@ffff9811c92c6a00 x1876700048984576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763361.284524:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048984576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763361.284542:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763361.284546:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763361.284549:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763361.539625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763362.179755:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763362.563601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763363.203697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763363.587608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763364.227677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763364.611567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763365.251654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763365.635551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763366.275607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763366.403556:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763366.403566:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763366.403571:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763366.403583:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2815 00000100:00080000:1.0:1789763366.403595:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763366.403603:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763366.403648:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048984704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763366.403672:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763366.403675:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763366.403676:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763366.403679:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2815 00000100:00080000:1.0:1789763366.403682:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763366.403684:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763366.403693:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763366.403703:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048984832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763366.403712:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763366.403716:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763366.403718:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763366.403721:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048984704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763366.403723:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2815 00000100:00080000:1.0:1789763366.403728:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763366.403733:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763366.403760:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763366.403771:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048984960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763366.403779:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763366.403782:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763366.403783:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763366.403786:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2815 00000100:00080000:1.0:1789763366.403790:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763366.403793:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763366.403797:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763366.403804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048985088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763366.403810:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763366.403816:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763366.403820:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763366.403822:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763366.403825:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2815 00000100:00080000:1.0:1789763366.403828:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763366.403830:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763366.403834:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763366.403840:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048985216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763366.403863:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763366/real 1789763366] req@ffff9811c92c5f80 x1876700048984704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763366.403882:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048984704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763366.403891:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763366.403894:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763366.403897:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048984704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763366.403925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048984832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763366.403952:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763366.403964:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763366/real 1789763366] req@ffff9811c92c4a80 x1876700048984832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763366.403973:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048984832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763366.403980:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763366.403983:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763366.403988:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048984832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763366.404016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763366.404040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763366.404051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763366/real 1789763366] req@ffff9811c92c4380 x1876700048984960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763366.404060:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048984960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763366.404066:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763366.404068:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763366.404071:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048984960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763366.404075:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763366.404090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763366.404096:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763366/real 1789763366] req@ffff9811c92c6680 x1876700048985088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763366.404104:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048985088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763366.404109:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763366.404111:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763366.404113:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763366.404116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763366.404128:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763366.404134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763366/real 1789763366] req@ffff9811c92c5c00 x1876700048985216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763366.404149:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048985216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763366.404155:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763366.404157:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763366.404159:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763366.659551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763367.299691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763367.683611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763368.323660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763368.707595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763369.347619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763369.731590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763370.371692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763370.755589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763371.395703:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763371.523627:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763371.523640:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763371.523647:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763371.523663:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2821 00000100:00080000:0.0:1789763371.523677:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763371.523688:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763371.523756:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048985344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763371.523797:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763371.523803:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763371.523806:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763371.523813:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2821 00000100:00080000:0.0:1789763371.523819:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763371.523826:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763371.523838:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763371.523861:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048985472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763371.523869:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048985344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763371.523890:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763371.523897:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763371.523900:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763371.523908:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2821 00000100:00080000:0.0:1789763371.523916:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763371.523922:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763371.523930:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763371.523949:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048985600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763371.523989:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763371.523993:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763371.523996:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763371.524014:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2821 00000100:00080000:0.0:1789763371.524022:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763371.524025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763371.524033:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763371.524044:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763371.524062:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048985728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763371.524075:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763371.524079:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763371.524082:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763371.524089:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2821 00000100:00080000:0.0:1789763371.524095:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763371.524103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763371/real 1789763371] req@ffff9811c2e29f80 x1876700048985344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763371.524105:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763371.524117:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763371.524139:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048985856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763371.524150:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048985344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763371.524171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763371.524178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763371.524184:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048985344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763371.524220:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048985472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763371.524252:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763371.524268:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763371/real 1789763371] req@ffff9811c2e28a80 x1876700048985472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763371.524284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048985472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763371.524295:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763371.524299:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763371.524304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048985472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763371.524311:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763371.524333:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763371.524347:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763371/real 1789763371] req@ffff9811c2e28380 x1876700048985600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763371.524406:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048985600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763371.524425:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763371.524433:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763371.524441:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763371.524455:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763371.524544:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763371.524567:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763371/real 1789763371] req@ffff9811c2e2aa00 x1876700048985728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763371.524595:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048985728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763371.524611:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763371.524618:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763371.524623:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763371.524635:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763371.524659:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763371.524673:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763371/real 1789763371] req@ffff9811c2e2b100 x1876700048985856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763371.524687:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048985856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763371.524720:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763371.524728:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763371.524736:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048985856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763371.779564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763372.419688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763372.803581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763373.443717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763373.827610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763374.467706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763374.851581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763375.491751:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763375.875665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763376.515705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763376.643602:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763376.643613:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763376.643618:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763376.643631:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2826 00000100:00080000:1.0:1789763376.643644:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763376.643653:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763376.643702:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048985984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763376.643741:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763376.643746:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763376.643749:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763376.643753:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2826 00000100:00080000:1.0:1789763376.643758:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763376.643762:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763376.643772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763376.643786:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048986112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763376.643831:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763376.643836:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763376.643839:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763376.643846:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2826 00000100:00080000:1.0:1789763376.643851:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763376.643856:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763376.643864:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763376.643877:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048985984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763376.643883:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048986240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763376.643902:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763376.643906:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763376.643909:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763376.643913:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2826 00000100:00080000:1.0:1789763376.643947:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763376.643953:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763376.643960:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763376.643977:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048986368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763376.643991:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763376.643996:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763376.644003:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763376.644004:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763376.644013:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2826 00000100:00080000:1.0:1789763376.644036:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763376.644043:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763376.644051:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763376.644069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763376/real 1789763376] req@ffff9811c92c7480 x1876700048985984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763376.644075:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048986496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763376.644139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048985984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763376.644157:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763376.644163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763376.644170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048985984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763376.644179:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048986112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763376.644206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763376.644221:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763376/real 1789763376] req@ffff9811c92c4e00 x1876700048986112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763376.644236:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048986112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763376.644268:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763376.644272:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763376.644277:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048986112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763376.644309:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763376.644335:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763376.644349:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763376/real 1789763376] req@ffff9811c92c4000 x1876700048986240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763376.644363:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048986240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763376.644373:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763376.644377:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763376.644381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763376.644388:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763376.644406:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763376.644417:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763376/real 1789763376] req@ffff9811c92c5500 x1876700048986368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763376.644430:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048986368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763376.644439:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763376.644443:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763376.644455:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763376.644507:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763376.644538:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763376.644553:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763376/real 1789763376] req@ffff9811c92c6d80 x1876700048986496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763376.644569:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048986496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763376.644578:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763376.644582:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763376.644586:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763376.899598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763377.539739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763377.923645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763378.563723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763378.947574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763379.587707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763379.971607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763380.611756:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763380.995610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763381.635754:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763381.763665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763381.763676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763381.763682:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763381.763694:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2831 00000100:00080000:3.0:1789763381.763708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763381.763717:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763381.763766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700048986624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763381.763806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763381.763811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763381.763814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763381.763818:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2831 00000100:00080000:3.0:1789763381.763844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763381.763865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763381.763878:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763381.763951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700048986752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763381.763975:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763381.763982:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048986624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763381.763985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763381.763993:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763381.764007:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2831 00000100:00080000:3.0:1789763381.764015:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763381.764026:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763381.764037:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763381.764074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048986880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763381.764099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763381.764104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763381.764107:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763381.764112:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2831 00000100:00080000:3.0:1789763381.764118:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763381.764126:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789763381.764136:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763381.764145:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763381.764169:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048987008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763381.764200:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763381.764214:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763381.764216:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763381.764223:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2831 00000100:00080000:3.0:1789763381.764230:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763381.764244:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763381.764246:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763381/real 1789763381] req@ffff9811e1a22680 x1876700048986624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763381.764282:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763381.764302:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700048986624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763381.764308:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048987136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763381.764370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763381.764380:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763381.764388:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048986624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763381.764429:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048986752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763381.764458:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763381.764603:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763381/real 1789763381] req@ffff9811e1a21c00 x1876700048986752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763381.764621:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700048986752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763381.764633:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763381.764637:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763381.764642:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048986752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763381.764662:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763381.764691:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763381.764703:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763381/real 1789763381] req@ffff9811e1a21880 x1876700048986880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763381.764717:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048986880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763381.764727:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763381.764731:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763381.764734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048986880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763381.764741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763381.764764:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763381.764775:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763381/real 1789763381] req@ffff9811e1a22300 x1876700048987008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763381.764789:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048987008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763381.764809:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763381.764813:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763381.764817:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763381.764824:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763381.764850:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763381.764862:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763381/real 1789763381] req@ffff9811e1a21180 x1876700048987136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763381.764877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048987136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763381.764915:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763381.764919:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763381.764951:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763382.019581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763382.659687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763383.043569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763383.683700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763384.067544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763384.707674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763385.091643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763385.731702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763386.115628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763386.755663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763386.883688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763386.883701:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763386.883707:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763386.883721:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2836 00000100:00080000:2.0:1789763386.883745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763386.883755:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763386.883805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048987264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763386.883906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763386.883912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763386.883914:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763386.883920:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2836 00000100:00080000:2.0:1789763386.883926:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763386.883932:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763386.883945:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763386.883965:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048987392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763386.883982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763386.883987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763386.883990:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763386.883995:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2836 00000100:00080000:2.0:1789763386.883999:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763386.884009:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763386.884022:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048987264:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763386.884023:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763386.884041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048987520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763386.884055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763386.884059:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763386.884062:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763386.884066:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2836 00000100:00080000:2.0:1789763386.884084:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763386.884094:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763386.884104:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789763386.884136:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763386.884139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048987648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763386.884176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763386.884183:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763386.884187:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763386.884197:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2836 00000100:00080000:2.0:1789763386.884206:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763386.884214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763386/real 1789763386] req@ffff9811d8639180 x1876700048987264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763386.884217:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763386.884229:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763386.884268:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048987776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763386.884296:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048987264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763386.884324:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763386.884333:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763386.884342:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048987264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763386.884357:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048987392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763386.884408:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763386.884429:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763386/real 1789763386] req@ffff9811d863bb80 x1876700048987392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763386.884446:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048987392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763386.884537:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763386.884542:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763386.884547:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048987392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763386.884591:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763386.884632:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763386.884656:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763386/real 1789763386] req@ffff9811d863ad80 x1876700048987520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763386.884682:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048987520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763386.884700:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763386.884707:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763386.884712:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763386.884720:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763386.884764:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763386.884786:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763386/real 1789763386] req@ffff9811d8639c00 x1876700048987648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763386.884812:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048987648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763386.884829:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763386.884836:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763386.884906:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763386.884915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763386.884946:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763386.884961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763386/real 1789763386] req@ffff9811d8639f80 x1876700048987776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763386.884980:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048987776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763386.884997:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763386.885016:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763386.885023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048987776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763387.139558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763387.779784:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763388.163602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763388.803614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763389.187582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763389.827746:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763390.211613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763390.851680:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763391.235616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763391.875692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763392.003579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763392.003635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763392.003640:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763392.003655:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2841 00000100:00080000:0.0:1789763392.003670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763392.003680:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763392.003748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048987904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763392.003784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763392.003789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763392.003791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763392.003795:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2841 00000100:00080000:0.0:1789763392.003799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763392.003803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763392.003849:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763392.003863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048988032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763392.003873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763392.003876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763392.003878:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763392.003882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2841 00000100:00080000:0.0:1789763392.003886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763392.003892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763392.003896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763392.003906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048988160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763392.003919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763392.003923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763392.003925:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763392.003928:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2841 00000100:00080000:0.0:1789763392.003932:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763392.003935:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763392.003940:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763392.003951:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048987904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763392.003960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048988288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763392.003980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763392.003983:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763392.003984:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763392.003986:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2841 00000100:00080000:0.0:1789763392.003989:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763392.003993:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763392.003997:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763392.004015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048988416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763392.004118:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048988032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763392.004153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763392.004196:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763392.004221:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763392.259636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763392.899671:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763393.283591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763393.923684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763394.307596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763394.947667:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763395.076029:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:1.0:1789763395.076102:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763395.076130:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763395.076204:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763395.076221:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763395.076271:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:1.0:1789763395.076289:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789763395.076320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763392/real 1789763395] req@ffff9811c2e2a680 x1876700048988288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.076371:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048988288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.076405:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.076416:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763395.076445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763395.076529:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763392/real 1789763395] req@ffff9811c2e2bb80 x1876700048988416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.076565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048988416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.076583:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.076593:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763395.076603:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763395.076619:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763392/real 1789763395] req@ffff9811c2e29c00 x1876700048987904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.076654:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048987904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.076672:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.076681:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763395.076689:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048987904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763395.076718:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763392/real 1789763395] req@ffff9811c2e29880 x1876700048988032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.076747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048988032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.076764:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.076771:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763395.076823:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048988032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763395.076840:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763392/real 1789763395] req@ffff9811c2e29180 x1876700048988160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763395.076871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048988160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763395.076908:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763395.076916:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763395.076924:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763395.331584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763395.971677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763396.355572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763396.995669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763397.123627:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763397.123640:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763397.123671:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763397.123685:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2846 00000100:00080000:0.0:1789763397.123700:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763397.123710:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763397.123796:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700048988544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763397.123848:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763397.123853:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763397.123856:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763397.123862:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2846 00000100:00080000:0.0:1789763397.123867:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763397.123871:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763397.123883:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763397.123902:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700048988672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763397.123917:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763397.123922:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763397.123924:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763397.123929:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2846 00000100:00080000:0.0:1789763397.123932:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763397.123942:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763397.123950:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763397.123960:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048988544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763397.123973:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700048988800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763397.124051:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763397.124058:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763397.124061:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763397.124069:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2846 00000100:00080000:0.0:1789763397.124077:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763397.124088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763397.124089:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763397.124116:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763397.124142:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700048988928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763397.124158:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763397.124186:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763397.124191:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763397.124195:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763397/real 1789763397] req@ffff9811c2e2ad80 x1876700048988544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763397.124202:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2846 00000100:00080000:0.0:1789763397.124213:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763397.124226:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763397.124239:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763397.124251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700048988544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763397.124264:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700048989056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763397.124280:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763397.124294:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763397.124304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048988544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763397.124317:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048988672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763397.124361:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763397.124386:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763397/real 1789763397] req@ffff9811c2e28e00 x1876700048988672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763397.124436:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700048988672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763397.124453:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763397.124459:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763397.124521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048988672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763397.124561:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763397.124601:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763397.124622:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763397/real 1789763397] req@ffff9811c2e28700 x1876700048988800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763397.124645:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700048988800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763397.124661:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763397.124666:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763397.124672:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763397.124683:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763397.124719:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763397.124738:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763397/real 1789763397] req@ffff9811c2e29180 x1876700048988928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763397.124759:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700048988928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763397.124823:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763397.124832:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763397.124841:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048988928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763397.124853:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763397.124896:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763397.124931:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763397/real 1789763397] req@ffff9811c2e29880 x1876700048989056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763397.124961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700048989056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763397.124975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763397.124981:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763397.124986:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763397.379548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763398.019731:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763398.403572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763399.043786:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763399.427571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763400.067786:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763400.451536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763401.091634:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763401.475554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763402.115723:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763402.243608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763402.243622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763402.243628:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763402.243640:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2851 00000100:00080000:1.0:1789763402.243654:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763402.243663:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763402.243714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700048989184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763402.243777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763402.243782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763402.243785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763402.243791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2851 00000100:00080000:1.0:1789763402.243796:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763402.243802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763402.243815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763402.243845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700048989312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763402.243862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763402.243867:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763402.243870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763402.243875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2851 00000100:00080000:1.0:1789763402.243879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763402.243887:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048989184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763402.243912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763402.243922:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763402.243942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700048989440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763402.243957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763402.243963:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763402.243966:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763402.243970:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2851 00000100:00080000:1.0:1789763402.243977:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763402.243984:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763402.243990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763402.244000:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763402.244018:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700048989568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763402.244032:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763402.244036:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763402.244039:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763402.244044:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2851 00000100:00080000:1.0:1789763402.244052:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763402.244053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763402/real 1789763402] req@ffff9811c92c5180 x1876700048989184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.244065:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763402.244079:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763402.244100:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700048989696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763402.244103:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700048989184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763402.244145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763402.244164:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763402.244173:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048989184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763402.244185:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048989312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763402.244217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763402.244234:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763402/real 1789763402] req@ffff9811c92c7b80 x1876700048989312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763402.244250:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700048989312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763402.244261:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763402.244265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763402.244269:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048989312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763402.244303:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763402.244340:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763402.244363:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763402/real 1789763402] req@ffff9811c92c5880 x1876700048989440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763402.244389:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700048989440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763402.244406:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763402.244410:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763402.244414:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763402.244423:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763402.244509:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763402.244527:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763402/real 1789763402] req@ffff9811c92c6d80 x1876700048989568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763402.244544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700048989568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763402.244554:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763402.244558:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763402.244562:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763402.244569:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763402.244593:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763402.244604:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763402/real 1789763402] req@ffff9811c92c5500 x1876700048989696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763402.244617:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700048989696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763402.244627:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763402.244630:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763402.244633:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048989696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763402.499502:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763403.139683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763403.523535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763404.163707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763404.547564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763405.187731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763405.571550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763406.211675:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763406.595601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763407.235827:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763407.363653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763407.363733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763407.363740:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763407.363761:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2856 00000100:00080000:1.0:1789763407.363781:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763407.363793:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763407.363882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700048989824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763407.363951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763407.363958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763407.363961:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763407.363972:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2856 00000100:00080000:1.0:1789763407.363979:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763407.363988:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763407.364007:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763407.364036:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048989824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763407.364037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700048989952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763407.364079:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763407.364085:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763407.364090:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763407.364104:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2856 00000100:00080000:1.0:1789763407.364110:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763407.364115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763407.364122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763407.364139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700048990080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763407.364166:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763407.364171:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763407.364174:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763407.364182:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2856 00000100:00080000:1.0:1789763407.364189:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763407.364194:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763407.364201:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763407.364216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048990208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763407.364231:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763407.364235:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763407.364238:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763407.364244:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2856 00000100:00080000:1.0:1789763407.364248:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763407.364254:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763407.364263:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763407.364282:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048990336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763407.364569:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763407.364752:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763407/real 1789763407] req@ffff9811c92c4000 x1876700048989824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763407.364781:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700048989824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763407.364796:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763407.364802:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763407.364808:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048989824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763407.364842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048989952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763407.364877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763407.364895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763407/real 1789763407] req@ffff9811c92c4e00 x1876700048989952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763407.364911:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700048989952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763407.364922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763407.364926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763407.364944:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048989952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763407.364954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763407.364982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763407.364995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763407/real 1789763407] req@ffff9811c92c7480 x1876700048990080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763407.365011:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700048990080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763407.365021:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763407.365024:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763407.365028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763407.365039:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763407.365101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763407.365127:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763407/real 1789763407] req@ffff9811c92c5c00 x1876700048990208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763407.365147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048990208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763407.365158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763407.365162:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763407.365166:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763407.365175:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763407.365206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763407.365243:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763407/real 1789763407] req@ffff9811c92c6680 x1876700048990336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763407.365269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048990336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763407.365286:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763407.365293:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763407.365300:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763407.619595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763408.259744:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763408.643531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763409.283663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763409.667649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763410.307722:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763410.691611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763411.331635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763411.715584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763412.355714:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763412.483545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763412.483553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763412.483557:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763412.483565:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2862 00000100:00080000:1.0:1789763412.483592:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763412.483601:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763412.483644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048990464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763412.483684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763412.483696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763412.483698:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763412.483701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2862 00000100:00080000:1.0:1789763412.483704:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763412.483707:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763412.483714:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763412.483725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048990592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763412.483733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763412.483736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763412.483737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763412.483741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2862 00000100:00080000:1.0:1789763412.483743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763412.483748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048990464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763412.483749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763412.483761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763412.483771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048990720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763412.483779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763412.483786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763412.483788:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763412.483791:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2862 00000100:00080000:1.0:1789763412.483794:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763412.483798:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763412.483802:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763412.483810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048990848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763412.483817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763412.483821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763412.483823:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763412.483823:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763412.483830:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2862 00000100:00080000:1.0:1789763412.483834:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763412.483836:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763412.483840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763412.483847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048990976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763412.483866:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763412/real 1789763412] req@ffff9811c92c4380 x1876700048990464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763412.483881:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048990464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763412.483889:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763412.483892:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763412.483895:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048990464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763412.483900:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048990592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763412.483913:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763412.483930:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763412/real 1789763412] req@ffff9811c92c4a80 x1876700048990592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763412.483939:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048990592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763412.483944:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763412.483947:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763412.483949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048990592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763412.483967:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763412.483980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763412.483987:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763412/real 1789763412] req@ffff9811c92c5f80 x1876700048990720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763412.483994:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048990720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763412.483999:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763412.484001:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763412.484003:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763412.484006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763412.484016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763412.484022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763412/real 1789763412] req@ffff9811c92c6a00 x1876700048990848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763412.484029:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048990848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763412.484039:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763412.484041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763412.484043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763412.484047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763412.484060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763412.484066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763412/real 1789763412] req@ffff9811c92c6300 x1876700048990976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763412.484073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048990976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763412.484078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763412.484080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763412.484081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048990976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763412.739623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763413.379720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763413.763635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763414.403750:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763414.787621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763415.427796:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763415.811614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763416.451725:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763416.835602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763417.475747:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763417.603598:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763417.603611:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763417.603634:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763417.603648:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2867 00000100:00080000:2.0:1789763417.603674:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763417.603707:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763417.603761:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048991104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763417.603802:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763417.603807:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763417.603809:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763417.603827:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2867 00000100:00080000:2.0:1789763417.603835:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763417.603842:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763417.603855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763417.603875:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048991104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763417.603879:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048991232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763417.603903:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763417.603908:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763417.603911:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763417.603916:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2867 00000100:00080000:2.0:1789763417.603920:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763417.603925:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763417.603931:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763417.603945:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048991360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763417.603957:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763417.603962:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763417.603975:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763417.603982:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2867 00000800:00000100:3.0:1789763417.603992:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763417.603993:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763417.604007:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763417.604017:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763417.604043:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048991488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763417.604057:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763417.604061:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763417.604064:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763417.604073:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2867 00000100:00100000:3.0:1789763417.604074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763417/real 1789763417] req@ffff9811d863b100 x1876700048991104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763417.604079:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763417.604091:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763417.604116:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763417.604124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048991104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763417.604149:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048991616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763417.604158:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763417.604176:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763417.604220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048991104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763417.604266:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048991232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763417.604355:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763417.604386:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763417/real 1789763417] req@ffff9811d8639880 x1876700048991232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763417.604425:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048991232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763417.604437:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763417.604442:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763417.604447:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048991232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763417.604456:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763417.604573:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763417.604605:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763417/real 1789763417] req@ffff9811d8639500 x1876700048991360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763417.604656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048991360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763417.604675:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763417.604681:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763417.604689:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763417.604717:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763417.604765:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763417.604791:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763417/real 1789763417] req@ffff9811d863a680 x1876700048991488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763417.604811:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048991488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763417.604822:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763417.604826:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763417.604830:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763417.604838:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763417.604865:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763417.604878:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763417/real 1789763417] req@ffff9811d8638a80 x1876700048991616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763417.604892:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048991616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763417.604901:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763417.604904:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763417.604908:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048991616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763417.859638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763418.499764:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763418.883636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763419.523718:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763419.907589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763420.547806:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763420.931625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763421.571753:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763421.955738:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763422.595849:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763422.723630:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763422.723648:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763422.723656:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763422.723672:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2872 00000100:00080000:2.0:1789763422.723690:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763422.723700:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763422.723750:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048991744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763422.723820:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763422.723825:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763422.723828:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763422.723833:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2872 00000100:00080000:2.0:1789763422.723839:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763422.723844:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763422.723856:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763422.723874:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048991872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763422.723893:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763422.723900:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763422.723904:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763422.723912:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2872 00000100:00080000:2.0:1789763422.723920:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763422.723932:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763422.723947:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763422.723953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048991744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763422.723978:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048992000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763422.724007:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763422.724011:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763422.724015:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763422.724021:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2872 00000100:00080000:2.0:1789763422.724067:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763422.724075:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763422.724087:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763422.724089:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763422.724127:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048992128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763422.724151:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763422.724158:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763422.724164:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763422.724175:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2872 00000100:00080000:2.0:1789763422.724185:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763422.724189:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763422/real 1789763422] req@ffff9811d863b800 x1876700048991744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763422.724200:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763422.724223:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763422.724249:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048992256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763422.724258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048991744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763422.724302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763422.724314:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763422.724324:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048991744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763422.724340:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048991872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763422.724382:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763422.724402:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763422/real 1789763422] req@ffff9811d863a300 x1876700048991872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763422.724418:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048991872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763422.724430:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763422.724435:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763422.724439:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048991872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763422.724538:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763422.724566:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763422.724613:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763422/real 1789763422] req@ffff9811d863b480 x1876700048992000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763422.724656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048992000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763422.724667:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763422.724672:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763422.724676:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763422.724685:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763422.724718:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763422.724732:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763422/real 1789763422] req@ffff9811d863aa00 x1876700048992128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763422.724748:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048992128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763422.724757:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763422.724761:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763422.724764:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763422.724772:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763422.724791:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763422.724802:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763422/real 1789763422] req@ffff9811d8638000 x1876700048992256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763422.724816:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048992256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763422.724844:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763422.724849:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763422.724853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763422.979670:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763423.619818:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763424.003644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763424.643789:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763425.027653:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763425.667791:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763426.051698:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763426.691743:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763427.075659:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763427.715729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763427.843610:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763427.843623:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763427.843628:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763427.843642:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2877 00000100:00080000:2.0:1789763427.843656:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763427.843666:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763427.843716:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700048992384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763427.843759:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763427.843789:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763427.843791:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763427.843797:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2877 00000100:00080000:2.0:1789763427.843803:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763427.843809:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763427.843822:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763427.843843:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700048992512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763427.843875:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763427.843880:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048992384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763427.843882:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763427.843893:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763427.843914:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2877 00000100:00080000:2.0:1789763427.843919:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763427.843924:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763427.843933:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763427.843954:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700048992640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763427.843976:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763427.843985:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763427.843989:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763427.843997:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2877 00000100:00080000:2.0:1789763427.844008:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763427.844018:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789763427.844027:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789763427.844050:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763427.844083:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700048992768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763427.844101:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763427.844105:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763427/real 1789763427] req@ffff9811d8638380 x1876700048992384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763427.844108:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763427.844116:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763427.844144:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2877 00000100:00080000:2.0:1789763427.844170:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763427.844179:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763427.844187:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700048992384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763427.844193:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763427.844229:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763427.844236:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763427.844244:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700048992896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763427.844245:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048992384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763427.844317:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048992512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763427.844352:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763427.844368:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763427/real 1789763427] req@ffff9811d8638e00 x1876700048992512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763427.844385:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700048992512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763427.844418:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763427.844423:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763427.844428:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048992512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763427.844437:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763427.844511:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763427.844529:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763427/real 1789763427] req@ffff9811d8638700 x1876700048992640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763427.844575:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700048992640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763427.844587:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763427.844591:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763427.844604:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763427.844614:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763427.844647:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763427.844662:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763427/real 1789763427] req@ffff9811d8638000 x1876700048992768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763427.844677:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700048992768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763427.844686:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763427.844690:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763427.844705:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763427.844714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763427.844752:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763427.844775:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763427/real 1789763427] req@ffff9811d863aa00 x1876700048992896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763427.844802:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700048992896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763427.844819:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763427.844825:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763427.844832:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048992896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763428.099654:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763428.739765:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763429.123605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763429.763755:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763430.147622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763430.787756:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763431.171596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763431.811757:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763432.195620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763432.835718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763432.963666:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763432.963686:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763432.963694:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763432.963714:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2882 00000100:00080000:0.0:1789763432.963736:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763432.963789:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763432.963864:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700048993024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763432.963925:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763432.963935:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763432.963939:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763432.963949:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2882 00000100:00080000:0.0:1789763432.963959:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763432.963967:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763432.963987:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763432.964018:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700048993152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763432.964073:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763432.964075:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048993024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763432.964091:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763432.964099:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763432.964113:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2882 00000100:00080000:0.0:1789763432.964125:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763432.964137:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763432.964151:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763432.964185:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700048993280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763432.964189:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763432.964222:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763432.964230:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763432.964249:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763432.964263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763432/real 1789763432] req@ffff9811c2e29c00 x1876700048993024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763432.964283:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2882 00000100:00100000:3.0:1789763432.964294:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700048993024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763432.964302:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763432.964319:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763432.964331:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763432.964337:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:0.0:1789763432.964353:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763432.964372:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048993024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763432.964401:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048993408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763432.964414:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048993152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763432.964579:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763432.964586:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763432.964600:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763432.964607:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763432.964610:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763432/real 1789763432] req@ffff9811c2e2bb80 x1876700048993152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763432.964638:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2882 00000100:00080000:0.0:1789763432.964654:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763432.964655:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700048993152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763432.964678:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763432.964680:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763432.964696:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763432.964703:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048993152:192.168.203.113@tcp:38:kworker.0 00000008:00100000:0.0:1789763432.964705:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763432.964714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763432.964753:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048993536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763432.964753:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763432.964773:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763432/real 1789763432] req@ffff9811c2e2a680 x1876700048993280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763432.964814:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700048993280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763432.964825:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763432.964831:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763432.964835:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763432.964860:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763432.964882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763432.964909:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763432/real 1789763432] req@ffff9811c2e2b100 x1876700048993408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763432.964924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048993408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763432.964935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763432.964938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763432.964942:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763432.964949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763432.964971:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763432.964981:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763432/real 1789763432] req@ffff9811c2e2aa00 x1876700048993536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763432.964995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048993536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763432.965004:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763432.965007:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763432.965011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763433.219656:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763433.859758:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763434.243629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763434.883661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763435.267647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763435.907679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763436.291602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763436.931733:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763437.315563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763437.955703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763438.083643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763438.083660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763438.083668:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763438.083685:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2887 00000100:00080000:2.0:1789763438.083701:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763438.083714:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763438.083763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700048993664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763438.083801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763438.083809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763438.083813:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763438.083820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2887 00000100:00080000:2.0:1789763438.083849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763438.083858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763438.083867:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048993664:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763438.083873:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763438.083899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700048993792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763438.083921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763438.083927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763438.083930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763438.083937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2887 00000100:00080000:2.0:1789763438.083942:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763438.083948:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763438.083956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763438.083975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700048993920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763438.083975:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763438.083993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763438.083999:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763438.084002:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763438.084008:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2887 00000100:00080000:2.0:1789763438.084014:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763438.084020:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763438.084028:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763438.084045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048994048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763438.084048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763438/real 1789763438] req@ffff9811d863b480 x1876700048993664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763438.084081:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700048993664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763438.084087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763438.084095:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763438.084098:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763438.084101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763438.084105:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2887 00000100:00080000:3.0:1789763438.084108:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763438.084112:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763438.084115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048993664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763438.084119:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763438.084127:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763438.084145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048994176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763438.084153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048993792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763438.084215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763438.084241:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763438/real 1789763438] req@ffff9811d863a300 x1876700048993792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763438.084260:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700048993792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763438.084271:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763438.084275:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763438.084279:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048993792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763438.084287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763438.084315:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763438.084328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763438/real 1789763438] req@ffff9811d863b800 x1876700048993920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763438.084342:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700048993920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763438.084370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763438.084374:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763438.084379:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048993920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763438.084386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763438.084413:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763438.084425:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763438/real 1789763438] req@ffff9811d8638a80 x1876700048994048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763438.084439:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048994048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763438.084448:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763438.084452:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763438.084455:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763438.084552:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763438.084598:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763438.084629:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763438/real 1789763438] req@ffff9811d863a680 x1876700048994176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763438.084646:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048994176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763438.084656:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763438.084660:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763438.084674:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763438.339553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763438.979642:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763439.363560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763440.003625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763440.387630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763441.027705:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763441.411580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763442.051741:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763442.435624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763443.075692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763443.203604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763443.203619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763443.203625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763443.203641:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2892 00000100:00080000:2.0:1789763443.203659:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763443.203671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763443.203726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048994304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763443.203767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763443.203772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763443.203775:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763443.203779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2892 00000100:00080000:2.0:1789763443.203784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763443.203788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763443.203826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763443.203861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048994432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763443.203872:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048994304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763443.203883:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763443.203889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763443.203892:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763443.203900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2892 00000100:00080000:2.0:1789763443.203906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763443.203927:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763443.203935:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763443.203953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048994560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763443.203969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763443.203975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763443.203979:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763443.203987:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2892 00000100:00080000:2.0:1789763443.203996:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763443.204002:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763443.204010:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763443.204044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763443.204047:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048994688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763443.204061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763443.204065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763443.204075:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763443.204081:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2892 00000100:00080000:2.0:1789763443.204087:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763443.204092:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763443.204101:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763443.204115:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048994816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763443.204587:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763443/real 1789763443] req@ffff9811d8639500 x1876700048994304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763443.204630:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048994304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763443.204663:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763443.204672:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763443.204679:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048994304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763443.204724:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048994432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763443.204775:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763443.204877:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763443/real 1789763443] req@ffff9811d8639880 x1876700048994432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763443.204895:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048994432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763443.204907:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763443.204922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763443.204927:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048994432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763443.204935:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763443.204966:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763443.204980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763443/real 1789763443] req@ffff9811d863b100 x1876700048994560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763443.204995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048994560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763443.205005:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763443.205009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763443.205013:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763443.205019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763443.205057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763443.205066:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763443/real 1789763443] req@ffff9811d8639f80 x1876700048994688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763443.205077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048994688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763443.205086:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763443.205089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763443.205092:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763443.205097:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763443.205125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763443.205139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763443/real 1789763443] req@ffff9811d8639c00 x1876700048994816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763443.205151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048994816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763443.205159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763443.205161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763443.205164:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048994816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763443.459569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763444.099720:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763444.483604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763445.123714:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763445.507595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763446.147689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763446.531572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763447.171622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763447.555613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763448.195616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763448.323576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763448.323587:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763448.323593:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763448.323606:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2897 00000100:00080000:0.0:1789763448.323647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763448.323657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763448.323708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048994944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763448.323740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763448.323744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763448.323746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763448.323750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2897 00000100:00080000:0.0:1789763448.323753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763448.323756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763448.323769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763448.323780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048995072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763448.323793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763448.323799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763448.323801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048994944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763448.323802:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763448.323812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2897 00000100:00080000:0.0:1789763448.323816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763448.323819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763448.323823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763448.323833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048995200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763448.323840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763448.323850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763448.323852:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763448.323856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2897 00000100:00080000:0.0:1789763448.323860:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763448.323865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763448.323871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763448.323881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048995328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763448.323891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763448.323894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763448.323895:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763448.323899:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2897 00000100:00080000:0.0:1789763448.323902:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763448.323904:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763448.323910:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763448.323915:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763448.323925:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048995456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763448.323968:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763448/real 1789763448] req@ffff9811c2e28380 x1876700048994944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763448.323988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048994944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763448.324001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763448.324005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763448.324009:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048994944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763448.324036:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048995072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763448.324070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763448.324082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763448/real 1789763448] req@ffff9811c2e28a80 x1876700048995072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763448.324093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048995072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763448.324101:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763448.324104:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763448.324107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048995072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763448.324112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763448.324134:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763448.324148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763448/real 1789763448] req@ffff9811c2e29f80 x1876700048995200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763448.324164:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048995200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763448.324176:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763448.324181:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763448.324186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763448.324194:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763448.324219:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763448.324229:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763448/real 1789763448] req@ffff9811c2e2b800 x1876700048995328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763448.324255:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048995328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763448.324262:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763448.324265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763448.324268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763448.324272:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763448.324290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763448.324298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763448/real 1789763448] req@ffff9811c2e2a300 x1876700048995456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763448.324328:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048995456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763448.324338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763448.324341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763448.324345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763448.579619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763449.219637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763449.603572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763450.243690:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763450.627558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763451.267583:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763451.651608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763452.291686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763452.675568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763453.315592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763453.443652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763453.443665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763453.443689:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763453.443701:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2903 00000100:00080000:3.0:1789763453.443715:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763453.443725:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763453.443770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700048995584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763453.443826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763453.443831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763453.443834:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763453.443839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2903 00000100:00080000:3.0:1789763453.443844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763453.443848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763453.443860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763453.443877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700048995712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763453.443895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763453.443902:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048995584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763453.443903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763453.443910:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763453.443933:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2903 00000100:00080000:3.0:1789763453.443938:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763453.443944:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763453.443951:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763453.443970:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700048995840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763453.444001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763453.444010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763453.444012:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763453.444019:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763453.444027:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2903 00000100:00080000:3.0:1789763453.444036:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763453.444042:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763453.444051:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763453.444071:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700048995968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763453.444079:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763453/real 1789763453] req@ffff9811e1a22d80 x1876700048995584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763453.444095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763453.444110:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763453.444117:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763453.444125:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2903 00000100:00080000:3.0:1789763453.444129:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763453.444135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700048995584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763453.444140:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763453.444153:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763453.444168:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700048996096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763453.444178:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763453.444194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763453.444201:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048995584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763453.444234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048995712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763453.444264:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763453.444282:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763453/real 1789763453] req@ffff9811e1a22a00 x1876700048995712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763453.444309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700048995712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763453.444345:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763453.444377:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763453.444386:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048995712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763453.444399:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763453.444445:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763453.444528:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763453/real 1789763453] req@ffff9811e1a21180 x1876700048995840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763453.444554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700048995840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763453.444565:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763453.444569:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763453.444573:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763453.444582:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763453.444614:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763453.444629:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763453/real 1789763453] req@ffff9811e1a22300 x1876700048995968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763453.444643:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700048995968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763453.444652:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763453.444656:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763453.444671:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048995968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763453.444679:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763453.444704:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763453.444716:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763453/real 1789763453] req@ffff9811e1a21880 x1876700048996096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763453.444730:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700048996096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763453.444740:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763453.444744:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763453.444748:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763453.699590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763454.339657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763454.723628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763455.363668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763455.747596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763456.387727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763456.771573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763457.411620:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763457.795601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763458.435723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763458.563641:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763458.563655:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763458.563661:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763458.563675:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2908 00000100:00080000:2.0:1789763458.563688:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763458.563698:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763458.563745:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700048996224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763458.563786:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763458.563791:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763458.563794:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763458.563799:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2908 00000100:00080000:2.0:1789763458.563803:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763458.563844:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763458.563855:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763458.563873:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048996224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763458.563874:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700048996352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763458.563929:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763458.563936:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763458.563941:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763458.563950:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2908 00000100:00080000:2.0:1789763458.563960:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763458.563969:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763458.563981:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763458.564018:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700048996480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763458.564039:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763458.564044:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763458.564047:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763458.564054:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2908 00000100:00080000:2.0:1789763458.564063:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763458.564076:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763458.564077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048996352:192.168.203.113@tcp:38:kworker.0 00000008:00100000:2.0:1789763458.564091:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763458.564119:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700048996608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763458.564137:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763458.564142:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996480:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789763458.564172:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763458.564175:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763458.564181:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2908 00000100:00080000:2.0:1789763458.564188:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763458.564194:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763458.564201:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763458.564204:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763458.564241:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700048996736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763458.564283:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763458.819576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763459.459662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763459.843524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763460.483663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763460.867595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763461.507640:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763461.635988:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789763461.636076:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763461.636108:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763461.636178:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763461.636197:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763461.636216:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763461.636235:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:3.0:1789763461.636269:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763458/real 1789763461] req@ffff9811d8639c00 x1876700048996608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763461.636346:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700048996608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763461.636372:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763461.636382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763461.636391:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763461.636407:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763458/real 1789763461] req@ffff9811d863bb80 x1876700048996352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763461.636436:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700048996352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763461.636524:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763461.636535:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763461.636544:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048996352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763461.636560:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763458/real 1789763461] req@ffff9811d8639180 x1876700048996480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763461.636589:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700048996480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763461.636604:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763461.636612:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763461.636618:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763461.636632:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763458/real 1789763461] req@ffff9811d863ad80 x1876700048996224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763461.636658:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700048996224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763461.636674:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763461.636682:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763461.636688:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048996224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763461.636741:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763458/real 1789763461] req@ffff9811d8639f80 x1876700048996736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763461.636763:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700048996736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763461.636793:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763461.636800:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763461.636807:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048996736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763461.891577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763462.531724:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763462.915574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763463.555687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763463.683586:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763463.683602:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763463.683608:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763463.683625:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2913 00000100:00080000:2.0:1789763463.683643:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763463.683655:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763463.683716:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700048996864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763463.683768:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763463.683777:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763463.683781:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763463.683789:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2913 00000100:00080000:2.0:1789763463.683798:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763463.683809:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763463.683830:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763463.683846:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048996864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763463.683860:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700048996992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763463.683909:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763463.683914:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763463.683918:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763463.683927:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2913 00000100:00080000:2.0:1789763463.683944:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763463.683953:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763463.683964:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763463.683965:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763463.684002:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700048997120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763463.684026:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763463.684034:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763463.684039:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763463.684048:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2913 00000100:00080000:2.0:1789763463.684059:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763463.684060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763463/real 1789763463] req@ffff9811d863b100 x1876700048996864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763463.684077:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763463.684089:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763463.684115:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700048997248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763463.684138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700048996864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763463.684141:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763463.684158:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763463.684167:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763463.684185:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2913 00000100:00080000:3.0:1789763463.684188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763463.684204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763463.684209:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763463.684224:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763463.684232:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048996864:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763463.684241:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763463.684275:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700048997376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763463.684321:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048996992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763463.684387:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763463.684419:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763463/real 1789763463] req@ffff9811d8639880 x1876700048996992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763463.684446:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700048996992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763463.684547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763463.684556:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763463.684564:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048996992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763463.684576:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763463.684627:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763463.684651:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763463/real 1789763463] req@ffff9811d8639500 x1876700048997120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763463.684672:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700048997120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763463.684688:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763463.684694:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763463.684701:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763463.684714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763463.684755:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763463.684778:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763463/real 1789763463] req@ffff9811d863a680 x1876700048997248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763463.685680:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700048997248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763463.685708:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763463.685718:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763463.685751:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763463.685818:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763463.685922:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763463.685949:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763463/real 1789763463] req@ffff9811d8638a80 x1876700048997376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763463.685962:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700048997376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763463.685975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763463.685980:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763463.685985:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763463.939551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763464.579685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763464.963584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763465.603653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763465.987546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763466.627661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763467.011603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763467.651657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763468.035540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763468.675626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763468.803570:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763468.803580:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763468.803584:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763468.803596:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2918 00000100:00080000:0.0:1789763468.803608:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763468.803615:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763468.803657:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700048997504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763468.803683:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763468.803686:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763468.803688:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763468.803691:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2918 00000100:00080000:0.0:1789763468.803693:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763468.803696:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763468.803704:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763468.803713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700048997632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763468.803721:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763468.803724:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763468.803725:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763468.803727:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2918 00000100:00080000:0.0:1789763468.803729:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763468.803731:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763468.803734:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763468.803759:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700048997760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763468.803766:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763468.803768:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763468.803769:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763468.803774:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2918 00000100:00080000:0.0:1789763468.803777:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763468.803781:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763468.803784:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763468.803794:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048997504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763468.803795:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700048997888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763468.803818:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763468.803820:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763468.803822:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763468.803828:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2918 00000100:00080000:0.0:1789763468.803830:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763468.803834:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763468.803837:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763468.803851:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700048998016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763468.803877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763468.803918:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763468/real 1789763468] req@ffff9811c2e2b480 x1876700048997504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763468.803931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700048997504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763468.803956:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763468.803959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763468.803962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048997504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763468.803968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048997632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763468.803987:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763468.803995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763468/real 1789763468] req@ffff9811c2e29500 x1876700048997632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763468.804008:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700048997632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763468.804016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763468.804020:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763468.804023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048997632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763468.804030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763468.804052:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763468.804064:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763468/real 1789763468] req@ffff9811c2e28000 x1876700048997760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763468.804076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700048997760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763468.804082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763468.804084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763468.804086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763468.804114:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763468.804137:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763468.804149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763468/real 1789763468] req@ffff9811c2e2a300 x1876700048997888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763468.804161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700048997888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763468.804169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763468.804173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763468.804175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048997888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763468.804180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763468.804195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763468.804202:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763468/real 1789763468] req@ffff9811c2e2b800 x1876700048998016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763468.804209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700048998016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763468.804214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763468.804216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763468.804218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763469.059552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763469.699622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763470.083547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763470.723684:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763471.107647:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763471.747627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763472.131632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763472.771642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763473.155581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763473.795648:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763473.923605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763473.923617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763473.923622:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763473.923636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2923 00000100:00080000:1.0:1789763473.923650:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763473.923659:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763473.923706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700048998144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763473.923743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763473.923747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763473.923750:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763473.923755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2923 00000100:00080000:1.0:1789763473.923759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763473.923763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763473.923774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763473.923789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700048998272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763473.923824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763473.923828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763473.923830:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763473.923836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2923 00000100:00080000:1.0:1789763473.923841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763473.923846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763473.923852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763473.923866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700048998400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763473.923879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763473.923883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763473.923886:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763473.923891:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2923 00000100:00080000:1.0:1789763473.923894:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763473.923903:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763473.923909:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048998144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789763473.923911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763473.923941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700048998528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763473.923956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763473.923960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763473.923963:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763473.923969:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2923 00000100:00080000:1.0:1789763473.923984:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763473.923990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763473.923997:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763473.924016:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700048998656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763473.924022:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763473.924086:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763473/real 1789763473] req@ffff9811c92c4700 x1876700048998144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763473.924110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700048998144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763473.924124:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763473.924129:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763473.924134:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048998144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763473.924143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048998272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763473.924176:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763473.924191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763473/real 1789763473] req@ffff9811c92c7800 x1876700048998272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763473.924241:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700048998272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763473.924253:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763473.924258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763473.924263:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048998272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763473.924271:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763473.924316:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763473.924333:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763473/real 1789763473] req@ffff9811c92c7100 x1876700048998400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763473.924348:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700048998400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763473.924357:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763473.924361:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763473.924365:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763473.924396:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763473.924418:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763473.924430:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763473/real 1789763473] req@ffff9811c92c6300 x1876700048998528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763473.924444:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700048998528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763473.924454:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763473.924457:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763473.924461:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763473.924514:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763473.924539:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763473.924554:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763473/real 1789763473] req@ffff9811c92c6a00 x1876700048998656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763473.924590:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700048998656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763473.924610:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763473.924613:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763473.924617:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048998656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763474.179540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763474.819617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763475.203574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763475.844633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763476.227576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763476.868616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763477.251619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763477.891674:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763478.275624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763478.916576:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763479.043616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763479.043628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763479.043633:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763479.043663:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2928 00000100:00080000:1.0:1789763479.043675:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763479.043684:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763479.043728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700048998784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763479.043754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763479.043757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763479.043759:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763479.043765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2928 00000100:00080000:1.0:1789763479.043769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763479.043773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763479.043782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763479.043796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700048998912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763479.043806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763479.043822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763479.043824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763479.043830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2928 00000100:00080000:1.0:1789763479.043836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763479.043841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763479.043846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763479.043877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700048999040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763479.043890:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763479.043895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763479.043904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763479.043904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048998784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763479.043922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2928 00000100:00080000:1.0:1789763479.043927:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763479.043934:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763479.043943:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763479.043956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700048999168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763479.043980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763479.043985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763479.043986:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763479.043989:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2928 00000100:00080000:1.0:1789763479.044077:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763479.044085:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:3.0:1789763479.044088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763479.044101:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763479.044135:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700048999296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763479.044194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763479/real 1789763479] req@ffff9811c92c5f80 x1876700048998784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763479.044231:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700048998784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763479.044242:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763479.044246:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763479.044250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048998784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763479.044270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048998912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763479.044298:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763479.044310:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763479/real 1789763479] req@ffff9811c92c4a80 x1876700048998912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763479.044321:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700048998912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763479.044328:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763479.044331:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763479.044335:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048998912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763479.044359:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763479.044375:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763479.044384:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763479/real 1789763479] req@ffff9811c92c4380 x1876700048999040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763479.044393:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700048999040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763479.044400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763479.044402:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763479.044404:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763479.044409:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763479.044423:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763479.044431:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763479/real 1789763479] req@ffff9811c92c6680 x1876700048999168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763479.044448:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700048999168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763479.044455:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763479.044457:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763479.044460:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763479.044465:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763479.044520:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763479.044535:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763479/real 1789763479] req@ffff9811c92c5c00 x1876700048999296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763479.044546:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700048999296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763479.044553:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763479.044556:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763479.044559:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763479.299607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763479.939729:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763480.323571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763480.963682:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763481.347565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763481.987755:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763482.371593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763483.011736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763483.395601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763484.035790:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763484.163677:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763484.163690:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763484.163695:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763484.163733:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2933 00000100:00080000:0.0:1789763484.163746:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763484.163756:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763484.163802:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700048999424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763484.163846:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763484.163851:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763484.163854:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763484.163859:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2933 00000100:00080000:0.0:1789763484.163863:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763484.163868:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763484.163881:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763484.163908:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700048999552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763484.163910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048999424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763484.163949:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763484.163954:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763484.163956:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763484.163961:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2933 00000100:00080000:0.0:1789763484.163966:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763484.163971:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763484.163976:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763484.163988:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700048999680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763484.164018:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763484.164022:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763484.164025:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763484.164030:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2933 00000100:00080000:0.0:1789763484.164035:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763484.164046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763484.164052:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763484.164061:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763484.164078:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700048999808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763484.164089:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763484.164094:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763484.164096:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763484.164106:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2933 00000100:00080000:0.0:1789763484.164112:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763484.164117:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763484.164124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763484/real 1789763484] req@ffff9811c2e29f80 x1876700048999424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763484.164151:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763484.164165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700048999424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763484.164184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763484.164185:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700048999936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763484.164202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763484.164216:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700048999424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763484.164268:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048999552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763484.164299:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763484.164312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763484/real 1789763484] req@ffff9811c2e28a80 x1876700048999552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763484.164322:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700048999552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763484.164338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763484.164342:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763484.164346:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700048999552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763484.164355:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763484.164382:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763484.164395:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763484/real 1789763484] req@ffff9811c2e28380 x1876700048999680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763484.164408:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700048999680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763484.164432:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763484.164436:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763484.164440:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763484.164449:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763484.164510:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763484.164530:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763484/real 1789763484] req@ffff9811c2e2aa00 x1876700048999808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763484.164544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700048999808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763484.164550:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763484.164553:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763484.164555:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763484.164560:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763484.164578:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763484.164586:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763484/real 1789763484] req@ffff9811c2e2b100 x1876700048999936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763484.164594:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700048999936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763484.164599:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763484.164602:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763484.164610:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700048999936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763484.419571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763485.059710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763485.443662:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763486.083792:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763486.467671:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763487.107848:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763487.491559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763488.131664:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763488.515629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763489.155706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763489.283637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763489.283653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763489.283661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763489.283678:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2938 00000100:00080000:0.0:1789763489.283722:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763489.283736:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763489.283792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049000064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763489.283861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763489.283870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763489.283874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763489.283883:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2938 00000100:00080000:0.0:1789763489.283891:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763489.283899:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763489.283916:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763489.283945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049000192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763489.283968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049000064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763489.283977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763489.283987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763489.283992:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763489.284003:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2938 00000100:00080000:0.0:1789763489.284013:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763489.284022:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763489.284033:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763489.284066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049000320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763489.284087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763489.284123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763489.284145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763489.284151:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763489.284161:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2938 00000100:00080000:0.0:1789763489.284173:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763489.284182:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763489/real 1789763489] req@ffff9811c2e2a680 x1876700049000064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763489.284190:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763489.284214:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763489.284238:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049000064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763489.284249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049000448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763489.284260:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763489.284275:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763489.284288:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049000064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763489.284294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763489.284304:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763489.284309:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763489.284319:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2938 00000100:00100000:3.0:1789763489.284327:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049000192:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763489.284332:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763489.284346:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763489.284358:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763489.284368:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763489.284401:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049000576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763489.284401:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763489/real 1789763489] req@ffff9811c2e2bb80 x1876700049000192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763489.284445:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049000192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763489.284501:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763489.284509:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763489.284529:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049000192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763489.284541:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763489.284581:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763489.284607:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763489/real 1789763489] req@ffff9811c2e29c00 x1876700049000320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763489.284633:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049000320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763489.284649:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763489.284656:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763489.284663:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763489.284675:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763489.284715:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763489.284732:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763489/real 1789763489] req@ffff9811c2e29880 x1876700049000448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763489.284747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049000448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763489.284757:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763489.284760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763489.284765:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763489.284786:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763489.284808:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763489.284845:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763489/real 1789763489] req@ffff9811c2e29180 x1876700049000576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763489.284869:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049000576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763489.284886:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763489.284893:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763489.284899:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763489.539597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763490.179786:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763490.563576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763491.203607:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763491.587587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763492.227733:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763492.611606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763493.251715:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763493.635601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763494.275662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763494.403752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763494.403786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763494.403793:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763494.403832:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2943 00000100:00080000:0.0:1789763494.403897:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763494.403912:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763494.404021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049000704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763494.404138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763494.404146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763494.404150:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763494.404159:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2943 00000100:00080000:0.0:1789763494.404168:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763494.404178:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763494.404199:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763494.404233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049000704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763494.404235:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049000832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763494.404270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763494.404278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763494.404282:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763494.404290:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2943 00000100:00080000:0.0:1789763494.404299:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763494.404309:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763494.404322:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763494.404349:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049000960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763494.404373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789763494.404376:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763494.404380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763494.404387:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763494.404394:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2943 00000100:00080000:0.0:1789763494.404414:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763494.404424:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763494.404437:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763494.404509:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763494/real 1789763494] req@ffff9811c2e28700 x1876700049000704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763494.404542:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049000704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763494.404559:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763494.404565:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763494.404572:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049000704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763494.404615:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049000832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763494.404642:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049001088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763494.404661:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763494.404689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763494.404699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789763494.404704:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763494/real 1789763494] req@ffff9811c2e28e00 x1876700049000832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763494.404746:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049000832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763494.404753:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763494.404760:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763494.404774:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763494.404779:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049000832:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763494.404784:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2943 00000100:00100000:3.0:1789763494.404790:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000960:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763494.404800:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763494.404828:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763494.404836:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763494.404852:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763494/real 1789763494] req@ffff9811c2e2ad80 x1876700049000960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763494.404870:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049000960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763494.404871:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763494.404891:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763494.404896:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763494.404900:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049000960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763494.404928:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763494.404941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049001216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763494.404959:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763494.404988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763494/real 1789763494] req@ffff9811c2e28700 x1876700049001088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763494.405006:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049001088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763494.405028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763494.405034:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763494.405038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763494.405080:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763494.405116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763494.405131:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763494/real 1789763494] req@ffff9811c2e29180 x1876700049001216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763494.405147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049001216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763494.405157:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763494.405161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763494.405165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763494.659590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763495.299686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763495.683653:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763496.323703:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763496.707609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763497.347621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763497.731552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763498.371641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763498.755610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763499.395731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763499.523675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763499.523688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763499.523693:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763499.523708:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2949 00000100:00080000:3.0:1789763499.523740:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763499.523749:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763499.523817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049001344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763499.523890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763499.523896:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763499.523898:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763499.523904:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2949 00000100:00080000:3.0:1789763499.523910:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763499.523933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763499.523944:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763499.523956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049001472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763499.523972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763499.523978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763499.523998:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763499.523999:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049001344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763499.524005:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2949 00000100:00080000:3.0:1789763499.524029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763499.524036:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763499.524043:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763499.524067:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049001600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763499.524076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763499.524079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763499.524081:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763499.524085:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2949 00000100:00080000:3.0:1789763499.524095:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763499.524100:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763499.524105:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763499.524116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049001728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763499.524124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763499.524127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763499.524129:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763499.524131:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2949 00000100:00080000:3.0:1789763499.524134:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763499.524136:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763499.524139:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763499.524149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049001856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763499.524154:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763499.524242:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763499/real 1789763499] req@ffff9811e1a21c00 x1876700049001344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763499.524270:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049001344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763499.524285:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763499.524291:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763499.524297:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049001344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763499.524335:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049001472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763499.524384:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763499.524412:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763499/real 1789763499] req@ffff9811e1a22680 x1876700049001472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763499.524508:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049001472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763499.524539:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763499.524547:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763499.524554:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049001472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763499.524568:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763499.524627:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763499.524653:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763499/real 1789763499] req@ffff9811e1a23480 x1876700049001600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763499.524685:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049001600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763499.524698:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763499.524702:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763499.524707:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763499.524726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763499.524758:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763499.524773:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763499/real 1789763499] req@ffff9811e1a21500 x1876700049001728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763499.524796:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049001728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763499.524835:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763499.524842:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763499.524862:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763499.524875:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763499.524924:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763499.524942:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763499/real 1789763499] req@ffff9811e1a20700 x1876700049001856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763499.524957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049001856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763499.524968:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763499.524972:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763499.524976:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049001856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763499.779572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763500.419742:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763500.803638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763501.443770:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763501.827654:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763502.467691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763502.851569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763503.491700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763503.875605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763504.515666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763504.643562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763504.643576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763504.643592:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763504.643608:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2954 00000100:00080000:3.0:1789763504.643624:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763504.643635:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763504.643692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049001984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763504.643745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763504.643771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763504.643774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763504.643781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2954 00000100:00080000:3.0:1789763504.643786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763504.643793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763504.643808:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763504.643829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049002112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763504.643842:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049001984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763504.643867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763504.643876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763504.643880:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763504.643891:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2954 00000100:00080000:3.0:1789763504.643899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763504.643909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763504.643920:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763504.643943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049002240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763504.643960:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763504.643997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763504.644006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763504.644010:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763504.644042:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2954 00000100:00080000:3.0:1789763504.644049:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763504.644055:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763504.644073:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763504.644074:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763504/real 1789763504] req@ffff9811e1a23100 x1876700049001984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763504.644114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049002368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763504.644154:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763504.644159:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763504.644162:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763504.644169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049001984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763504.644175:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2954 00000100:00080000:3.0:1789763504.644188:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763504.644201:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:2.0:1789763504.644204:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763504.644210:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763504.644215:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763504.644226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049001984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763504.644263:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049002496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763504.644287:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049002112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763504.644348:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763504.644372:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763504/real 1789763504] req@ffff9811e1a23800 x1876700049002112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763504.644397:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049002112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763504.644413:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763504.644419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763504.644426:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049002112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763504.644436:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763504.644466:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763504.644480:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763504/real 1789763504] req@ffff9811e1a20a80 x1876700049002240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763504.644494:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049002240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763504.644519:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763504.644523:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763504.644569:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763504.644578:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763504.644607:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763504.644621:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763504/real 1789763504] req@ffff9811e1a21f80 x1876700049002368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763504.644635:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049002368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763504.644644:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763504.644647:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763504.644651:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763504.644702:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763504.644729:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763504.644743:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763504/real 1789763504] req@ffff9811e1a20000 x1876700049002496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763504.644765:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049002496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763504.644797:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763504.644803:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763504.644825:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763504.899580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763505.539742:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763505.923616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763506.563709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763506.947654:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763507.587803:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763507.971630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763508.611670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763508.995574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763509.635802:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763509.763663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763509.763679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763509.763685:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763509.763703:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2959 00000100:00080000:3.0:1789763509.763721:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763509.763733:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763509.763799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049002624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763509.763858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763509.763864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763509.763867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763509.763872:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2959 00000100:00080000:3.0:1789763509.763927:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763509.763935:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763509.763996:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763509.764004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049002624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763509.764016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049002752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763509.764030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763509.764034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763509.764036:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763509.764058:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2959 00000100:00080000:3.0:1789763509.764063:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763509.764068:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763509.764076:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763509.764094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049002880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763509.764123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763509.764127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763509.764128:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763509.764132:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2959 00000100:00080000:3.0:1789763509.764147:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763509.764151:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763509.764157:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763509.764167:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049003008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763509.764175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763509.764195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763509.764199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763509.764202:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763509.764207:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2959 00000100:00080000:3.0:1789763509.764212:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763509.764215:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763509.764220:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763509.764233:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049003136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763509.764244:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763509/real 1789763509] req@ffff9811e1a20380 x1876700049002624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763509.764270:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049002624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763509.764283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763509.764288:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763509.764292:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049002624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763509.764320:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049002752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763509.764369:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763509.764384:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763509/real 1789763509] req@ffff9811e1a20e00 x1876700049002752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763509.764399:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049002752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763509.764409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763509.764425:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763509.764430:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049002752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763509.764438:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763509.764464:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763509.764476:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763509/real 1789763509] req@ffff9811e1a20000 x1876700049002880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763509.764490:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049002880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763509.764500:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763509.764503:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763509.764507:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049002880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763509.764556:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763509.764595:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763509.764609:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763509/real 1789763509] req@ffff9811e1a21f80 x1876700049003008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763509.764638:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049003008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763509.764646:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763509.764649:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763509.764652:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763509.764656:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763509.764685:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763509.764694:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763509/real 1789763509] req@ffff9811e1a20a80 x1876700049003136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763509.764705:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049003136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763509.764711:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763509.764713:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763509.764716:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763510.019600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763510.659710:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763511.043648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763511.683757:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763512.067615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763512.707712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763513.091626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763513.731727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763514.115610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763514.755716:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763514.883616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763514.883629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763514.883637:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763514.883650:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2964 00000100:00080000:3.0:1789763514.883681:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763514.883693:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763514.883746:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049003264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763514.883791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763514.883796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763514.883800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763514.883808:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2964 00000100:00080000:3.0:1789763514.883813:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763514.883819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763514.883863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763514.883897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049003392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763514.883898:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049003264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763514.883956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763514.883963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763514.883968:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763514.883980:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2964 00000100:00080000:3.0:1789763514.883988:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763514.883994:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763514.884007:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763514.884023:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049003520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763514.884041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763514.884047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763514.884050:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763514.884066:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763514.884145:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763514/real 1789763514] req@ffff9811e1a23800 x1876700049003264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763514.884187:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049003264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763514.884207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763514.884213:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763514.884220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049003264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763514.884288:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049003392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763514.884343:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763514.884378:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763514/real 1789763514] req@ffff9811e1a23100 x1876700049003392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763514.884407:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049003392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763514.884420:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763514.884422:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2964 00000100:00080000:0.0:1789763514.884436:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763514.884443:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763514.884451:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049003392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763514.884553:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763514.884610:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763514.884641:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763514.884643:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763514/real 1789763514] req@ffff9811e1a20700 x1876700049003520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763514.884665:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763514.884678:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049003520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763514.884694:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763514.884703:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049003648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763514.884709:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763514.884736:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003520:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763514.884751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763514.884759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763514.884772:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763514.884780:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003648:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763514.884790:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2964 00000100:00080000:3.0:1789763514.884809:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763514.884819:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763514.884831:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763514.884853:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763514.884875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049003776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763514.884895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763514/real 1789763514] req@ffff9811e1a21500 x1876700049003648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763514.884965:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049003648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763514.884977:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763514.884982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763514.884986:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763514.885007:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763514.885039:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763514.885054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763514/real 1789763514] req@ffff9811e1a23480 x1876700049003776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763514.885068:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049003776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763514.885078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763514.885083:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763514.885087:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049003776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763515.139613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763515.779776:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763516.163644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763516.803661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763517.187595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763517.827676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763518.211588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763518.851618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763519.235625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763519.875727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763520.003602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763520.003622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763520.003630:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763520.003650:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2969 00000100:00080000:3.0:1789763520.003669:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763520.003684:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763520.003753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049003904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763520.003801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763520.003809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763520.003812:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763520.003820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2969 00000100:00080000:3.0:1789763520.003827:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763520.003835:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763520.003850:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763520.003892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049004032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763520.003906:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049003904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763520.003940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763520.003945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763520.003948:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763520.003954:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2969 00000100:00080000:3.0:1789763520.003960:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763520.003966:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763520.003973:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763520.003989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049004160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763520.004001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763520.004005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763520.004007:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763520.004011:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2969 00000100:00080000:3.0:1789763520.004016:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763520.004023:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763520.004026:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763520.004034:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763520.004046:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049004288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763520.004058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763520.004062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763520.004064:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763520.004068:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2969 00000100:00080000:3.0:1789763520.004080:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763520.004086:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763520.004092:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763520.004110:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763520/real 1789763520] req@ffff9811e1a22680 x1876700049003904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763520.004111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049004416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763520.004178:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049003904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763520.004223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763520.004232:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763520.004239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049003904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763520.004281:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049004032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763520.004315:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763520.004333:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763520/real 1789763520] req@ffff9811e1a21c00 x1876700049004032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763520.004348:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049004032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763520.004358:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763520.004362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763520.004366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049004032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763520.004372:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763520.004406:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763520.004418:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763520/real 1789763520] req@ffff9811e1a21880 x1876700049004160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763520.004431:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049004160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763520.004439:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763520.004442:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763520.004446:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763520.004500:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763520.004530:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763520.004544:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763520/real 1789763520] req@ffff9811e1a22300 x1876700049004288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763520.004558:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049004288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763520.004567:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763520.004571:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763520.004574:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763520.004580:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763520.004597:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763520.004606:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763520/real 1789763520] req@ffff9811e1a21180 x1876700049004416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763520.004618:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049004416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763520.004627:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763520.004630:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763520.004633:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763520.259636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763520.899687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763521.283530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763521.923684:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763522.307559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763522.947588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763523.331537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763523.971686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763524.355603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763524.995635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763525.123694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763525.123711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763525.123719:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763525.123735:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2974 00000100:00080000:1.0:1789763525.123752:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763525.123766:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763525.123869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049004544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763525.123925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763525.123933:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763525.123936:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763525.123946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2974 00000100:00080000:1.0:1789763525.123955:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763525.123968:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763525.123988:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763525.124020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049004544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763525.124023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049004672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763525.124062:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763525.124068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763525.124072:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763525.124080:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2974 00000100:00080000:1.0:1789763525.124086:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763525.124091:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763525.124123:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763525.124142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049004800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763525.124156:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763525.124161:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763525.124164:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763525.124169:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2974 00000100:00080000:1.0:1789763525.124175:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763525.124180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763525.124186:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763525.124208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049004928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763525.124225:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763525.124229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763525.124231:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763525.124235:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2974 00000100:00080000:1.0:1789763525.124238:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763525.124243:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763525.124249:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763525.124269:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049004672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763525.124271:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049005056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763525.124376:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763525.124423:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763525.124541:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763525.379643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763526.019645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763526.403582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763527.043668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763527.427586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763528.067706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763528.195795:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763528.195872:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763528.195905:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763528.195968:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763528.195988:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763528.196008:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763528.196033:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763528.196051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763525/real 1789763528] req@ffff9811c92c4e00 x1876700049004672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763528.196091:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049004672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763528.196106:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763528.196113:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763528.196118:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049004672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763528.196128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763525/real 1789763528] req@ffff9811c92c4000 x1876700049004800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763528.196143:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049004800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763528.196152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763528.196156:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763528.196160:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763528.196167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763525/real 1789763528] req@ffff9811c92c5500 x1876700049004928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763528.196182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049004928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763528.196212:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763528.196217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763528.196222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049004928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763528.196229:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763525/real 1789763528] req@ffff9811c92c7480 x1876700049004544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763528.196245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049004544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763528.196259:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763528.196265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763528.196272:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049004544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763528.196317:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763525/real 1789763528] req@ffff9811c92c6d80 x1876700049005056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763528.196343:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049005056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763528.196359:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763528.196365:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763528.196371:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763528.451552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763529.091605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763529.475575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763530.115659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763530.243602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763530.243629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763530.243655:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763530.243667:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2979 00000100:00080000:1.0:1789763530.243700:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763530.243712:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763530.243780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049005184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763530.243869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763530.243873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763530.243874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763530.243878:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2979 00000100:00080000:1.0:1789763530.243881:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763530.243902:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763530.243917:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763530.243938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049005312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763530.243959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763530.243967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763530.243970:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763530.243987:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049005184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763530.243987:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2979 00000100:00080000:1.0:1789763530.244003:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763530.244026:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763530.244033:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763530.244050:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049005440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763530.244098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763530.244103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763530.244106:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763530.244111:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2979 00000800:00000100:0.0:1789763530.244131:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763530.244136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763530.244145:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763530.244154:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763530.244175:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049005568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763530.244195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763530.244201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763530.244204:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763530.244214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763530/real 1789763530] req@ffff9811c92c5880 x1876700049005184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763530.244217:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2979 00000100:00080000:1.0:1789763530.244228:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763530.244247:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763530.244262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763530.244281:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049005696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763530.244395:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049005184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763530.244420:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763530.244431:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763530.244440:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049005184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763530.244576:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049005312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763530.244625:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763530.244652:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763530/real 1789763530] req@ffff9811c92c7b80 x1876700049005312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763530.244677:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049005312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763530.244692:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763530.244698:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763530.244703:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049005312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763530.244726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763530.244766:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763530.244811:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763530/real 1789763530] req@ffff9811c92c5180 x1876700049005440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763530.244836:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049005440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763530.244874:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763530.244880:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763530.244886:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763530.244896:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763530.244940:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763530.244961:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763530/real 1789763530] req@ffff9811c92c6d80 x1876700049005568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763530.244980:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049005568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763530.244993:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763530.244997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763530.245002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763530.245012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763530.245059:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763530.245076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763530/real 1789763530] req@ffff9811c92c7480 x1876700049005696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763530.245093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049005696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763530.245105:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763530.245109:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763530.245125:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049005696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763530.499598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763531.139947:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763531.523601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763532.163710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763532.547539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763533.187637:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763533.571573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763534.211649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763534.595542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763535.235699:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763535.363729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763535.363742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763535.363748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763535.363787:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2984 00000100:00080000:2.0:1789763535.363802:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763535.363812:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763535.363864:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049005824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763535.363906:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763535.363912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763535.363914:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763535.363920:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2984 00000100:00080000:2.0:1789763535.363924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763535.363928:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763535.363939:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763535.363956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049005952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763535.363993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049005824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763535.364021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763535.364026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763535.364029:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763535.364037:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2984 00000100:00080000:2.0:1789763535.364044:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763535.364050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763535.364058:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763535.364077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049006080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763535.364094:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763535.364105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763535.364115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763535.364119:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763535.364131:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2984 00000100:00080000:2.0:1789763535.364139:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763535.364149:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763535.364162:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763535.364163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763535/real 1789763535] req@ffff9811d863b800 x1876700049005824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763535.364194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049006208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763535.364201:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049005824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763535.364224:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763535.364229:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763535.364234:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049005824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763535.364256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763535.364262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763535.364265:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763535.364272:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2984 00000100:00100000:0.0:1789763535.364281:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049005952:192.168.203.113@tcp:38:kworker.0 00000100:00080000:2.0:1789763535.364298:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763535.364308:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763535.364319:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763535.364326:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763535.364349:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049006336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763535.364352:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763535/real 1789763535] req@ffff9811d863a300 x1876700049005952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763535.364414:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049005952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763535.364427:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763535.364432:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763535.364437:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049005952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763535.364446:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763535.364564:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763535.364596:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763535/real 1789763535] req@ffff9811d863b480 x1876700049006080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763535.364613:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049006080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763535.364623:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763535.364627:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763535.364632:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763535.364639:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763535.364664:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763535.364676:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763535/real 1789763535] req@ffff9811d863aa00 x1876700049006208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763535.364690:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049006208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763535.364699:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763535.364702:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763535.364706:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763535.364725:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763535.364744:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763535.364791:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763535/real 1789763535] req@ffff9811d8638000 x1876700049006336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763535.364806:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049006336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763535.364826:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763535.364830:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763535.364834:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763535.619538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763536.259673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763536.643569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763537.283820:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763537.667601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763538.307627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763538.691579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763539.331708:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763539.715568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763540.355574:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763540.483689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763540.483699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763540.483703:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763540.483729:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2990 00000100:00080000:0.0:1789763540.483741:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763540.483749:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763540.483788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049006464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763540.483819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763540.483823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763540.483825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763540.483829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2990 00000100:00080000:0.0:1789763540.483832:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763540.483836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763540.483845:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763540.483858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049006592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763540.483871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763540.483877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763540.483877:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049006464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763540.483896:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763540.483902:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2990 00000100:00080000:0.0:1789763540.483906:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763540.483910:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763540.483917:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763540.483930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049006720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763540.483941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763540.483944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763540.483945:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763540.483949:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2990 00000100:00080000:0.0:1789763540.483954:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763540.483955:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763540.483963:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763540.483970:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763540.483981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049006848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763540.483990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763540.483993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763540.483995:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763540.484000:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2990 00000100:00080000:0.0:1789763540.484004:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763540.484012:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763540.484012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763540/real 1789763540] req@ffff9811c2e29880 x1876700049006464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763540.484021:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763540.484041:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049006464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763540.484045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049006976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763540.484072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763540.484080:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763540.484085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049006464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763540.484113:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049006592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763540.484135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763540.484163:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763540/real 1789763540] req@ffff9811c2e29c00 x1876700049006592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763540.484180:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049006592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763540.484213:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763540.484218:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763540.484222:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049006592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763540.484230:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763540.484257:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763540.484269:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763540/real 1789763540] req@ffff9811c2e2bb80 x1876700049006720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763540.484284:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049006720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763540.484293:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763540.484297:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763540.484301:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763540.484308:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763540.484326:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763540.484337:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763540/real 1789763540] req@ffff9811c2e2a680 x1876700049006848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763540.484350:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049006848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763540.484359:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763540.484363:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763540.484376:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763540.484384:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763540.484408:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763540.484420:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763540/real 1789763540] req@ffff9811c2e2b100 x1876700049006976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763540.484435:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049006976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763540.484445:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763540.484448:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763540.484452:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049006976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763540.739581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763541.379742:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763541.763555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763542.403641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763542.787571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763543.427601:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763543.811562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763544.451803:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763544.835579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763545.475717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763545.603656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763545.603698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763545.603706:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763545.603739:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 2995 00000100:00080000:3.0:1789763545.603762:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763545.603777:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763545.603844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049007104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763545.603895:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763545.603901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763545.603904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763545.603910:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 2995 00000100:00080000:3.0:1789763545.603926:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763545.603931:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763545.603945:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763545.603961:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049007232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763545.603977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763545.603992:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763545.603994:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763545.604026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2995 00000100:00080000:3.0:1789763545.604033:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763545.604045:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763545.604052:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049007104:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763545.604058:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763545.604099:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049007360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763545.604126:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763545.604143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763545.604148:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763545.604157:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2995 00000100:00080000:3.0:1789763545.604166:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763545.604177:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763545.604191:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763545.604216:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763545.604220:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049007488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763545.604245:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763545.604264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763545.604268:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763545.604277:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 2995 00000100:00080000:3.0:1789763545.604288:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763545.604300:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763545.604308:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763545/real 1789763545] req@ffff9811e1a22a00 x1876700049007104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763545.604316:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763545.604351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049007104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763545.604372:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763545.604377:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049007616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763545.604382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763545.604389:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049007104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763545.604422:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049007232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763545.604503:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763545.604609:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763545/real 1789763545] req@ffff9811e1a22d80 x1876700049007232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763545.604627:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049007232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763545.604641:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763545.604648:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763545.604655:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049007232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763545.604742:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763545.604802:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763545.604827:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763545/real 1789763545] req@ffff9811e1a21180 x1876700049007360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763545.604852:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049007360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763545.604869:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763545.604875:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763545.604882:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763545.604896:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763545.604948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763545.604967:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763545/real 1789763545] req@ffff9811e1a22300 x1876700049007488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763545.605033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049007488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763545.605044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763545.605048:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763545.605052:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763545.605060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763545.605090:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763545.605104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763545/real 1789763545] req@ffff9811e1a21880 x1876700049007616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763545.605118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049007616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763545.605145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763545.605149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763545.605152:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049007616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763545.859562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763546.499655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763546.883572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763547.523777:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763547.907586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763548.547617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763548.931570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763549.571689:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763549.955586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763550.595811:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763550.723602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763550.723617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763550.723624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763550.723657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3000 00000100:00080000:3.0:1789763550.723677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763550.723704:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763550.723776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049007744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763550.723850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763550.723859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763550.723863:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763550.723876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3000 00000100:00080000:3.0:1789763550.723889:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763550.723902:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763550.723926:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763550.723928:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049007744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763550.724005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049007872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763550.724035:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763550.724061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1789763550.724063:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763550.724111:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763550.724125:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3000 00000100:00080000:3.0:1789763550.724153:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763550.724168:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763550/real 1789763550] req@ffff9811e1a21c00 x1876700049007744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763550.724204:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049007744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763550.724253:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763550.724268:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763550.724277:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763550.724290:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763550.724344:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049007744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763550.724397:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049007872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763550.724445:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763550.724540:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763550/real 1789763550] req@ffff9811e1a22680 x1876700049007872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763550.724570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049008000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763550.724580:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049007872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763550.724619:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763550.724626:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763550.724635:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049007872:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763550.724730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763550.724739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763550.724744:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763550.724759:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3000 00000100:00080000:3.0:1789763550.724770:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763550.724779:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763550.724789:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008000:192.168.203.113@tcp:38:kworker.0 00000008:00100000:3.0:1789763550.724791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763550.724824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049008128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763550.724845:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763550.724847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763550.724862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763550.724865:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763550.724878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3000 00000100:00080000:3.0:1789763550.724887:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763550.724894:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763550.724900:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763550/real 1789763550] req@ffff9811e1a23480 x1876700049008000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763550.724905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763550.724920:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049008000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763550.724936:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763550.724940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049008256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763550.724949:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763550.724989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763550.725027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763550.725074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763550.725087:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763550/real 1789763550] req@ffff9811e1a21500 x1876700049008128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763550.725100:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049008128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763550.725109:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763550.725112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763550.725133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763550.725140:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763550.725158:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763550.725166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763550/real 1789763550] req@ffff9811e1a20700 x1876700049008256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763550.725177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049008256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763550.725185:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763550.725188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763550.725191:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763550.979579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763551.619812:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763552.003585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763552.643763:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763553.027594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763553.667744:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763554.051685:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763554.691743:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763555.075638:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763555.715739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763555.843636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763555.843649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763555.843655:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763555.843668:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3005 00000100:00080000:3.0:1789763555.843681:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763555.843691:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763555.843740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049008384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763555.843783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763555.843788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763555.843791:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763555.843796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3005 00000100:00080000:3.0:1789763555.843800:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763555.843804:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763555.843816:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763555.843830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049008512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763555.843868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763555.843873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763555.843875:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763555.843886:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049008384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763555.843886:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3005 00000100:00080000:3.0:1789763555.843898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763555.843906:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763555.843913:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763555.843932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049008640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763555.843946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763555.843951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763555.843953:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763555.843958:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3005 00000100:00080000:3.0:1789763555.843967:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763555.843972:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763555.843978:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763555.843989:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763555.843996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049008768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763555.844012:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763555.844017:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763555.844020:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763555.844025:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3005 00000100:00080000:3.0:1789763555.844040:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763555.844046:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763555.844055:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763555.844059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763555/real 1789763555] req@ffff9811e1a23100 x1876700049008384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763555.844074:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049008896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763555.844144:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049008384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763555.844160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763555.844166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763555.844171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049008384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763555.844181:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049008512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763555.844214:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763555.844231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763555/real 1789763555] req@ffff9811e1a23800 x1876700049008512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763555.844247:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049008512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763555.844257:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763555.844261:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763555.844265:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049008512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763555.844293:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763555.844313:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763555.844325:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763555/real 1789763555] req@ffff9811e1a20a80 x1876700049008640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763555.844339:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049008640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763555.844367:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763555.844371:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763555.844375:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763555.844383:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763555.844408:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763555.844420:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763555/real 1789763555] req@ffff9811e1a21f80 x1876700049008768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763555.844434:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049008768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763555.844444:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763555.844447:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763555.844450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763555.844457:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763555.844538:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763555.844560:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763555/real 1789763555] req@ffff9811e1a20000 x1876700049008896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763555.844586:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049008896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763555.844627:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763555.844634:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763555.844655:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049008896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763556.099571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763556.739754:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763557.123665:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763557.763604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763558.147609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763558.787759:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763559.171632:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763559.811717:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763560.195556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763560.835672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763560.963615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763560.963627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763560.963632:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763560.963646:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3010 00000100:00080000:3.0:1789763560.963670:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763560.963679:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763560.963726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049009024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763560.963765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763560.963770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763560.963773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763560.963778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3010 00000100:00080000:3.0:1789763560.963782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763560.963786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763560.963830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763560.963850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049009152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763560.963870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763560.963879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763560.963887:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763560.963889:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049009024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763560.963900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3010 00000100:00080000:3.0:1789763560.963920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763560.963927:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763560.963934:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763560.963953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049009280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763560.963966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763560.963973:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763560.963976:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763560.963981:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3010 00000100:00080000:3.0:1789763560.964032:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763560.964039:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763560.964046:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763560.964062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763560.964063:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049009408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763560.964095:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763560.964100:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763560.964110:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763560.964117:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3010 00000100:00080000:3.0:1789763560.964126:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763560.964132:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763560.964139:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763560.964163:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049009536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763560.964176:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763560/real 1789763560] req@ffff9811e1a20e00 x1876700049009024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763560.964219:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049009024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763560.964241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763560.964249:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763560.964257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049009024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763560.964307:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049009152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763560.964346:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763560.964364:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763560/real 1789763560] req@ffff9811e1a20380 x1876700049009152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763560.964380:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049009152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763560.964391:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763560.964395:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763560.964399:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049009152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763560.964431:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763560.964458:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763560.964470:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763560/real 1789763560] req@ffff9811e1a20000 x1876700049009280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763560.964492:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049009280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763560.964592:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763560.964598:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763560.964603:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763560.964611:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763560.964658:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763560.964695:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763560/real 1789763560] req@ffff9811e1a21f80 x1876700049009408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763560.964711:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049009408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763560.964722:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763560.964727:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763560.964730:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763560.964738:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763560.964761:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763560.964775:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763560/real 1789763560] req@ffff9811e1a20a80 x1876700049009536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763560.964832:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049009536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763560.964849:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763560.964856:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763560.964863:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763561.219620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763561.859680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763562.243601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763562.883659:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763563.267601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763563.907672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763564.291639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763564.931676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763565.315630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763565.955701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763566.083675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763566.083692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763566.083698:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763566.083715:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3015 00000100:00080000:0.0:1789763566.083733:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763566.083744:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763566.083806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049009664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763566.083857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763566.083865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763566.083869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763566.083877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3015 00000100:00080000:0.0:1789763566.083885:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763566.083896:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763566.083916:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763566.083932:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049009664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763566.083962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049009792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763566.083984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763566.083990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763566.083993:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763566.084000:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3015 00000100:00080000:0.0:1789763566.084007:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763566.084040:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763566.084047:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763566.084059:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763566.084089:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049009920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763566.084105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763566.084114:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763566/real 1789763566] req@ffff9811c2e2aa00 x1876700049009664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763566.084115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763566.084123:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763566.084139:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3015 00000100:00080000:0.0:1789763566.084157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763566.084164:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763566.084170:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049009664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763566.084174:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763566.084197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763566.084203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049010048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763566.084209:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763566.084225:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049009664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763566.084238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763566.084245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763566.084248:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763566.084254:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3015 00000100:00080000:0.0:1789763566.084258:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763566.084262:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763566.084268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763566.084275:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049009792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763566.084286:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049010176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763566.084319:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763566.084345:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763566/real 1789763566] req@ffff9811c2e28380 x1876700049009792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763566.084378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049009792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763566.084390:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763566.084394:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763566.084399:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049009792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763566.084407:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763566.084442:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763566.084508:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763566/real 1789763566] req@ffff9811c2e28a80 x1876700049009920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763566.084565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049009920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763566.084593:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763566.084599:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763566.084606:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049009920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763566.084620:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763566.084663:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763566.084687:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763566/real 1789763566] req@ffff9811c2e29f80 x1876700049010048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763566.084712:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049010048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763566.084743:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763566.084748:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763566.084752:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763566.084776:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763566.084806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763566.084820:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763566/real 1789763566] req@ffff9811c2e2b800 x1876700049010176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763566.084835:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049010176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763566.084846:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763566.084850:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763566.084853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763566.339572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763566.979714:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763567.363598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763568.003683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763568.387620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763569.027660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763569.411630:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763570.051756:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763570.435592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763571.075723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763571.203807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763571.203820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763571.203825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763571.203839:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3020 00000100:00080000:2.0:1789763571.203853:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763571.203862:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763571.203938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700049010304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763571.203980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763571.203984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763571.203986:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763571.203989:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3020 00000100:00080000:2.0:1789763571.203993:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763571.203997:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763571.204008:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763571.204021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700049010432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763571.204033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763571.204037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763571.204039:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763571.204043:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3020 00000100:00080000:2.0:1789763571.204046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763571.204051:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763571.204059:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763571.204063:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049010304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763571.204070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700049010560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763571.204081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763571.204084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763571.204126:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763571.204140:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3020 00000100:00080000:2.0:1789763571.204145:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763571.204151:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763571.204177:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763571.204199:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049010688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763571.204219:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763571.204220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763571.204234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763571.204240:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763571.204249:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3020 00000100:00080000:2.0:1789763571.204256:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763571.204273:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763571.204279:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763571.204297:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049010816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763571.204312:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763571/real 1789763571] req@ffff9811d8638700 x1876700049010304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763571.204368:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700049010304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763571.204386:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763571.204392:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763571.204398:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049010304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763571.204409:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049010432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763571.204442:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763571.204459:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763571/real 1789763571] req@ffff9811d8638e00 x1876700049010432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763571.204474:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700049010432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763571.204485:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763571.204569:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763571.204574:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049010432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763571.204604:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763571.204630:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763571.204654:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763571/real 1789763571] req@ffff9811d8638380 x1876700049010560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763571.204684:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700049010560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763571.204693:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763571.204696:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763571.204699:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763571.204705:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763571.204726:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763571.204736:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763571/real 1789763571] req@ffff9811d8638000 x1876700049010688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763571.204748:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049010688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763571.204755:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763571.204765:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763571.204768:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763571.204773:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763571.204788:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763571.204796:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763571/real 1789763571] req@ffff9811d863aa00 x1876700049010816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763571.204807:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049010816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763571.204828:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763571.204832:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763571.204835:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049010816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763571.459608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763572.099682:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763572.483625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763573.123703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763573.507626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763574.147761:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763574.531606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763575.171689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763575.555624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763576.195732:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763576.323585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763576.323623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763576.323631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763576.323651:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3025 00000100:00080000:2.0:1789763576.323673:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763576.323688:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763576.323782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049010944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763576.323864:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763576.323869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763576.323872:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763576.323878:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3025 00000100:00080000:2.0:1789763576.323883:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763576.323889:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763576.323904:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763576.323925:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049011072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763576.323942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763576.323951:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049010944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763576.323951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763576.323975:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763576.323983:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3025 00000100:00080000:2.0:1789763576.323987:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763576.323990:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763576.323995:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763576.324007:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049011200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763576.324017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763576.324020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763576.324022:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763576.324049:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3025 00000100:00080000:2.0:1789763576.324064:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763576.324070:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763576.324078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763576.324096:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763576.324105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049011328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763576.324127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763576.324134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763576.324138:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763576.324146:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3025 00000100:00080000:2.0:1789763576.324155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763576.324164:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763576.324174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763576/real 1789763576] req@ffff9811d863b480 x1876700049010944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763576.324175:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763576.324208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049010944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763576.324224:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763576.324227:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049011456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763576.324234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763576.324246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049010944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763576.324291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049011072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763576.324323:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763576.324338:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763576/real 1789763576] req@ffff9811d863a300 x1876700049011072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763576.324353:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049011072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763576.324363:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763576.324368:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763576.324372:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049011072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763576.324390:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763576.324427:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763576.324436:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763576/real 1789763576] req@ffff9811d863b800 x1876700049011200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763576.324447:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049011200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763576.324520:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763576.324525:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763576.324530:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763576.324538:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763576.324568:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763576.324583:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763576/real 1789763576] req@ffff9811d8638a80 x1876700049011328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763576.324615:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049011328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763576.324626:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763576.324630:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763576.324634:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763576.324641:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763576.324663:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763576.324675:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763576/real 1789763576] req@ffff9811d863a680 x1876700049011456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763576.324702:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049011456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763576.324713:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763576.324717:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763576.324721:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763576.579619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763577.219716:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763577.603671:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763578.243697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763578.627601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763579.267649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763579.651569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763580.291732:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763580.675604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763581.315635:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763581.443647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763581.443661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763581.443668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763581.443680:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3031 00000100:00080000:0.0:1789763581.443694:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763581.443704:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763581.443755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700049011584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763581.443810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763581.443818:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763581.443821:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763581.443829:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3031 00000100:00080000:0.0:1789763581.443837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763581.443844:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763581.443860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763581.443886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700049011712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763581.443951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763581.443962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763581.443963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049011584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763581.443968:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763581.444003:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3031 00000100:00080000:0.0:1789763581.444014:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763581.444024:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763581.444037:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763581.444058:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700049011840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763581.444075:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763581.444076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763581.444088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763581.444091:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763581.444098:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3031 00000100:00080000:0.0:1789763581.444105:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763581.444111:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763581.444118:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763581.444132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049011968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763581.444145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763581.444150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763581.444152:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763581.444162:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3031 00000100:00100000:2.0:1789763581.444166:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763581/real 1789763581] req@ffff9811c2e2a300 x1876700049011584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763581.444178:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763581.444185:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763581.444192:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763581.444213:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049012096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763581.444275:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700049011584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763581.444293:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763581.444298:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763581.444304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049011584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763581.444335:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049011712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763581.444380:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763581.444405:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763581/real 1789763581] req@ffff9811c2e28000 x1876700049011712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763581.444511:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700049011712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763581.444531:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763581.444539:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763581.444547:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049011712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763581.444560:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763581.444606:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763581.444629:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763581/real 1789763581] req@ffff9811c2e29500 x1876700049011840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763581.444645:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700049011840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763581.444674:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763581.444678:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763581.444683:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763581.444691:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763581.444721:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763581.444734:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763581/real 1789763581] req@ffff9811c2e2b480 x1876700049011968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763581.444749:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049011968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763581.444758:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763581.444762:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763581.444765:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049011968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763581.444772:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763581.444791:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763581.444801:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763581/real 1789763581] req@ffff9811c2e2b800 x1876700049012096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763581.444815:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049012096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763581.444824:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763581.444827:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763581.444839:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763581.699642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763582.339750:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763582.723575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763583.363630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763583.747551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763584.387757:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763584.771609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763585.411672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763585.795581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763586.435722:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763586.563617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763586.563633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763586.563640:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763586.563700:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3036 00000100:00080000:0.0:1789763586.563720:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763586.563733:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763586.563813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049012224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763586.563849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763586.563854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763586.563857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763586.563864:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3036 00000100:00080000:0.0:1789763586.563870:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763586.563875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763586.563891:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763586.563904:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049012224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763586.563933:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049012352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763586.563976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763586.564004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763586.564009:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763586.564018:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3036 00000100:00080000:0.0:1789763586.564027:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763586.564030:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763586.564046:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763586.564078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763586.564124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049012480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763586.564149:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763586/real 1789763586] req@ffff9811c2e29f80 x1876700049012224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763586.564150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763586.564193:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763586.564201:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763586.564211:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049012224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763586.564214:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3036 00000100:00080000:0.0:1789763586.564239:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763586.564243:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763586.564258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763586.564264:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763586.564271:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049012224:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763586.564287:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763586.564343:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049012352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763586.564354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049012608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763586.564410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763586.564419:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763586.564425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763586.564431:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763586.564443:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3036 00000100:00080000:0.0:1789763586.564452:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763586.564513:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763586/real 1789763586] req@ffff9811c2e28a80 x1876700049012352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763586.564525:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763586.564564:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049012352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763586.564569:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763586.564598:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763586.564624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049012736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763586.564631:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763586.564656:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049012352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763586.564707:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763586.564742:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763586.564777:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763586/real 1789763586] req@ffff9811c2e28380 x1876700049012480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763586.564799:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049012480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763586.564810:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763586.564816:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763586.564820:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763586.564857:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763586.564890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763586.564912:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763586/real 1789763586] req@ffff9811c2e2aa00 x1876700049012608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763586.564926:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049012608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763586.564935:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763586.564942:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763586.564949:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763586.564956:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763586.564983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763586.564993:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763586/real 1789763586] req@ffff9811c2e2b100 x1876700049012736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763586.565004:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049012736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763586.565012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763586.565014:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763586.565018:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049012736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763586.819613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763587.459661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763587.843579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763588.483815:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763588.867577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763589.507575:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763589.891610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763590.531701:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763590.915587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763591.555608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763591.683651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763591.683682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763591.683687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763591.683702:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3041 00000100:00080000:3.0:1789763591.683718:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763591.683768:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763591.683824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049012864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763591.683867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763591.683872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763591.683875:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763591.683880:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3041 00000100:00080000:3.0:1789763591.683885:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763591.683889:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763591.683902:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763591.683916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049012992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763591.683931:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763591.683936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763591.683938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763591.683943:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3041 00000100:00080000:3.0:1789763591.683947:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763591.683952:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763591.683958:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763591.683970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049013120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763591.683983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763591.683987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763591.684002:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763591.684004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049012864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763591.684012:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3041 00000100:00080000:3.0:1789763591.684022:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763591.684027:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763591.684052:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763591.684068:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049013248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763591.684079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763591.684083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763591.684086:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763591.684090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3041 00000100:00080000:3.0:1789763591.684111:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763591.684116:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763591.684121:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763591.684136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049013376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763591.684219:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049012992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763591.684292:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763591.684331:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763591.684397:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763591.939608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763592.579901:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763592.963563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763593.603611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763593.987584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763594.627612:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763594.755832:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:0.0:1789763594.755901:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763594.755926:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763594.756017:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763594.756036:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763594.756059:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:0.0:1789763594.756080:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789763594.756123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763591/real 1789763594] req@ffff9811e1a20700 x1876700049013120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763594.756175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049013120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763594.756198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763594.756207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763594.756215:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763594.756231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763591/real 1789763594] req@ffff9811e1a21500 x1876700049013248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763594.756259:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049013248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763594.756276:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763594.756283:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763594.756290:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763594.756303:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763591/real 1789763594] req@ffff9811e1a23480 x1876700049013376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763594.756364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049013376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763594.756382:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763594.756388:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763594.756421:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763594.756436:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763591/real 1789763594] req@ffff9811e1a23800 x1876700049012864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763594.756527:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049012864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763594.756544:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763594.756551:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763594.756559:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049012864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763594.756573:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763591/real 1789763594] req@ffff9811e1a23100 x1876700049012992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763594.756598:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049012992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763594.756612:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763594.756617:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763594.756624:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049012992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763595.011546:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763595.651641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763596.035567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763596.675728:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763596.803586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763596.803600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763596.803606:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763596.803619:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3046 00000100:00080000:3.0:1789763596.803633:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763596.803643:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763596.803704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049013504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763596.803768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763596.803773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763596.803781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763596.803787:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3046 00000100:00080000:3.0:1789763596.803792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763596.803798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763596.803810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763596.803830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049013632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763596.803845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763596.803851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763596.803858:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763596.803859:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049013504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763596.803867:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3046 00000100:00080000:3.0:1789763596.803877:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763596.803881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763596.803887:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763596.803907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049013760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763596.803919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763596.803924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763596.803926:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763596.803931:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3046 00000100:00080000:3.0:1789763596.803951:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763596.803958:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763596.803966:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763596.803979:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763596.803985:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049013888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763596.804009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763596.804014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763596.804016:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763596.804021:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3046 00000100:00080000:3.0:1789763596.804028:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763596.804037:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763596.804048:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763596.804058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763596/real 1789763596] req@ffff9811e1a22680 x1876700049013504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763596.804081:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049014016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763596.804169:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049013504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763596.804189:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763596.804197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763596.804205:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049013504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763596.804258:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049013632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763596.804343:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763596.804387:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763596/real 1789763596] req@ffff9811e1a21c00 x1876700049013632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763596.804410:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049013632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763596.804424:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763596.804431:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763596.804437:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049013632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763596.804465:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763596.804577:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763596.804628:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763596/real 1789763596] req@ffff9811e1a21880 x1876700049013760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763596.804652:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049013760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763596.804691:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763596.804697:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763596.804702:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763596.804714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763596.804757:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763596.804777:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763596/real 1789763596] req@ffff9811e1a22300 x1876700049013888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763596.804797:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049013888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763596.804810:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763596.804816:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763596.804822:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049013888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763596.804833:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763596.804869:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763596.804889:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763596/real 1789763596] req@ffff9811e1a21180 x1876700049014016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763596.804910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049014016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763596.804926:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763596.804931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763596.804951:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763597.059563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763597.699641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763598.083633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763598.723732:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763599.107554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763599.747640:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763600.131560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763600.771812:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763601.155527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763601.795652:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763601.923586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763601.923601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763601.923607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763601.923622:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3051 00000100:00080000:3.0:1789763601.923656:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763601.923671:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763601.923732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049014144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763601.923774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763601.923803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763601.923806:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763601.923813:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3051 00000100:00080000:3.0:1789763601.923820:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763601.923827:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763601.923841:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763601.923860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049014272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763601.923878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763601.923889:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049014144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763601.923890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763601.923904:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763601.923924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3051 00000100:00080000:3.0:1789763601.923929:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763601.923933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763601.923940:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763601.923957:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049014400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763601.923972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763601.923978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763601.923981:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763601.923984:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3051 00000100:00080000:3.0:1789763601.924009:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763601.924015:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789763601.924032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763601.924032:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763601.924057:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049014528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763601.924102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763601.924107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763601.924109:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763601.924119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3051 00000100:00100000:2.0:1789763601.924121:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763601/real 1789763601] req@ffff9811e1a22d80 x1876700049014144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763601.924130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763601.924150:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763601.924171:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763601.924184:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049014144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763601.924192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049014656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763601.924246:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763601.924257:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763601.924298:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049014144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763601.924364:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049014272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763601.924418:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763601.924443:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763601/real 1789763601] req@ffff9811e1a22a00 x1876700049014272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763601.924501:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049014272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763601.924541:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763601.924547:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763601.924552:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049014272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763601.924561:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763601.924596:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763601.924618:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763601/real 1789763601] req@ffff9811e1a21180 x1876700049014400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763601.924634:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049014400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763601.924645:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763601.924649:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763601.924653:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763601.924660:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763601.924682:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763601.924694:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763601/real 1789763601] req@ffff9811e1a22300 x1876700049014528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763601.924708:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049014528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763601.924717:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763601.924721:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763601.924735:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763601.924744:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763601.924771:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763601.924784:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763601/real 1789763601] req@ffff9811e1a21880 x1876700049014656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763601.924798:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049014656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763601.924808:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763601.924812:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763601.924816:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049014656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763602.179519:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763602.819695:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763603.203545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763603.843669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763604.227601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763604.867719:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763605.251569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763605.891699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763606.275576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763606.915756:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763607.043648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763607.043663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763607.043669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763607.043682:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3056 00000100:00080000:1.0:1789763607.043718:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763607.043742:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763607.043787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049014784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763607.043828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763607.043833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763607.043836:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763607.043842:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3056 00000100:00080000:1.0:1789763607.043846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763607.043851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763607.043863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763607.043888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049014912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763607.043908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763607.043913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763607.043916:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763607.043925:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049014784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763607.043927:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3056 00000100:00080000:1.0:1789763607.043940:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763607.043947:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763607.043953:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763607.043974:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049015040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763607.043987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763607.044000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763607.044003:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763607.044009:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3056 00000100:00080000:1.0:1789763607.044016:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763607.044021:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763607.044029:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763607.044031:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763607.044052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049015168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763607.044067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763607.044071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763607.044074:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763607.044080:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3056 00000100:00080000:1.0:1789763607.044086:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763607.044092:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763607.044100:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763607/real 1789763607] req@ffff9811c92c5500 x1876700049014784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789763607.044103:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763607.044126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049015296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763607.044141:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049014784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763607.044159:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763607.044179:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763607.044186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049014784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763607.044218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049014912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763607.044291:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763607.044310:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763607/real 1789763607] req@ffff9811c92c4000 x1876700049014912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763607.044327:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049014912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763607.044338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763607.044343:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763607.044347:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049014912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763607.044355:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763607.044376:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763607.044388:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763607/real 1789763607] req@ffff9811c92c4e00 x1876700049015040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763607.044402:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049015040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763607.044411:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763607.044415:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763607.044418:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763607.044425:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763607.044518:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763607.044535:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763607/real 1789763607] req@ffff9811c92c5c00 x1876700049015168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763607.044552:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049015168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763607.044562:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763607.044566:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763607.044570:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763607.044581:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763607.044621:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763607.044642:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763607/real 1789763607] req@ffff9811c92c6680 x1876700049015296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763607.044668:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049015296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763607.044685:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763607.044692:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763607.044698:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763607.299569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763607.939670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763608.323561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763608.963655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763609.347567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763609.987621:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763610.371605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763611.011702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763611.395569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763612.035692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763612.163596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763612.163612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763612.163619:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763612.163635:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3061 00000100:00080000:1.0:1789763612.163653:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763612.163694:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763612.163754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049015424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763612.163799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763612.163804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763612.163807:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763612.163812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3061 00000100:00080000:1.0:1789763612.163818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763612.163822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763612.163836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763612.163854:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049015552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763612.163873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763612.163879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763612.163885:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763612.163891:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049015424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763612.163896:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3061 00000100:00080000:1.0:1789763612.163911:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763612.163917:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763612.163922:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763612.163935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049015680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763612.163948:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763612.163952:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763612.163974:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763612.163984:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3061 00000100:00080000:1.0:1789763612.163993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763612.164003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763612.164019:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763612.164051:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763612.164055:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049015808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763612.164082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763612.164089:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763612.164094:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763612.164102:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3061 00000100:00080000:1.0:1789763612.164111:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763612.164117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763612/real 1789763612] req@ffff9811c92c4380 x1876700049015424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763612.164129:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763612.164157:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763612.164226:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049015424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763612.164227:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049015936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763612.164273:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763612.164288:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763612.164295:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049015424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763612.164333:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049015552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763612.164387:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763612.164407:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763612/real 1789763612] req@ffff9811c92c4a80 x1876700049015552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763612.164424:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049015552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763612.164436:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763612.164440:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763612.164445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049015552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763612.164452:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763612.164545:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763612.164563:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763612/real 1789763612] req@ffff9811c92c5f80 x1876700049015680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763612.164598:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049015680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763612.164608:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763612.164612:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763612.164616:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763612.164623:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763612.164663:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763612.164677:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763612/real 1789763612] req@ffff9811c92c6a00 x1876700049015808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763612.164691:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049015808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763612.164700:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763612.164704:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763612.164708:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763612.164714:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763612.164732:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763612.164742:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763612/real 1789763612] req@ffff9811c92c6300 x1876700049015936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763612.164754:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049015936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763612.164794:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763612.164802:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763612.164809:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049015936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763612.419609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763613.059727:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763613.443592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763614.083709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763614.467606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763615.107691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763615.491589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763616.131668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763616.515604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763617.155699:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763617.283641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763617.283659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763617.283668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763617.283686:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3066 00000100:00080000:3.0:1789763617.283720:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763617.283730:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763617.283777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049016064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763617.283814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763617.283836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763617.283839:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763617.283846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3066 00000100:00080000:3.0:1789763617.283852:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763617.283859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763617.283872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763617.283884:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049016064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763617.283893:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049016192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763617.283911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763617.283916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763617.283918:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763617.283924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3066 00000100:00080000:3.0:1789763617.283929:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763617.283933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763617.283938:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763617.283951:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049016320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763617.283962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763617.283967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763617.283969:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763617.283976:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763617.283983:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3066 00000100:00080000:3.0:1789763617.283992:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763617.283997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763617.284015:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763617.284039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763617/real 1789763617] req@ffff9811e1a21c00 x1876700049016064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763617.284045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049016448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763617.284085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763617.284088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049016064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763617.284099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763617.284106:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763617.284119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3066 00000100:00080000:2.0:1789763617.284127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763617.284128:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763617.284138:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763617.284169:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763617.284173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763617.284186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049016064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763617.284193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049016576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763617.284241:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049016192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763617.284304:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763617.284330:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763617/real 1789763617] req@ffff9811e1a22680 x1876700049016192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763617.284352:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049016192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763617.284374:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763617.284379:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763617.284384:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049016192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763617.284392:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763617.284423:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763617.284441:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763617/real 1789763617] req@ffff9811e1a23100 x1876700049016320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763617.284513:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049016320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763617.284527:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763617.284532:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763617.284536:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763617.284544:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763617.284576:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763617.284590:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763617/real 1789763617] req@ffff9811e1a23800 x1876700049016448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763617.284606:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049016448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763617.284621:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763617.284627:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763617.284647:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763617.284660:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763617.284703:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763617.284725:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763617/real 1789763617] req@ffff9811e1a23480 x1876700049016576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763617.284767:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049016576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763617.284784:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763617.284792:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763617.284798:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763617.539565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763618.179593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763618.563521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763619.203798:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763619.587591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763620.227628:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763620.611597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763621.251684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763621.635575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763622.275622:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763622.403565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763622.403576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763622.403582:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763622.403596:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3071 00000100:00080000:3.0:1789763622.403610:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763622.403619:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763622.403663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049016704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763622.403710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763622.403716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763622.403719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763622.403725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3071 00000100:00080000:3.0:1789763622.403752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763622.403763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763622.403781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763622.403806:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049016704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763622.403820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049016832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763622.403852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763622.403858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763622.403861:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763622.403868:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3071 00000100:00080000:3.0:1789763622.403875:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763622.403881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763622.403890:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763622.403903:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763622.403915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049016960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763622.403932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763622.403939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763622.403942:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763622.403989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763622/real 1789763622] req@ffff9811e1a21500 x1876700049016704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763622.404023:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049016704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763622.404040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763622.404045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763622.404054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049016704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763622.404057:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3071 00000100:00080000:3.0:1789763622.404071:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763622.404096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049016832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763622.404169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763622.404192:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763622/real 1789763622] req@ffff9811e1a20700 x1876700049016832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763622.404236:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049016832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763622.404255:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763622.404259:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763622.404268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763622.404270:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763622.404286:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049016832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763622.404298:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763622.404341:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763622.404367:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049017088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763622.404376:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763622/real 1789763622] req@ffff9811e1a20a80 x1876700049016960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763622.404404:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049016960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763622.404444:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763622.404452:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763622.404457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763622.404512:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049016960:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763622.404528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763622.404533:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763622.404544:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3071 00000100:00080000:3.0:1789763622.404552:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763622.404555:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017088:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763622.404566:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763622.404578:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763622.404601:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763622.404606:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049017216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763622.404625:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763622/real 1789763622] req@ffff9811e1a21f80 x1876700049017088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763622.404644:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049017088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763622.404674:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763622.404679:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763622.404683:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763622.404701:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763622.404738:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763622.404760:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763622/real 1789763622] req@ffff9811e1a20000 x1876700049017216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763622.404785:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049017216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763622.404802:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763622.404809:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763622.404815:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763622.659578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763623.299686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763623.683560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763624.323609:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763624.707566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763625.347730:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763625.731616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763626.371644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763626.755579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763627.395729:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763627.523565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763627.523575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763627.523582:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763627.523595:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3077 00000100:00080000:3.0:1789763627.523608:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763627.523619:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763627.523665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049017344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763627.523701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763627.523706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763627.523709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763627.523714:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3077 00000100:00080000:3.0:1789763627.523719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763627.523723:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763627.523755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763627.523773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049017472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763627.523792:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049017344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763627.523795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763627.523811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763627.523814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763627.523823:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3077 00000100:00080000:3.0:1789763627.523832:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763627.523841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763627.523852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763627.523878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049017600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763627.523896:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763627.523914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763627.523920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763627.523924:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763627.523932:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3077 00000100:00080000:3.0:1789763627.523939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763627.523944:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763627.523954:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763627.523963:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763627/real 1789763627] req@ffff9811e1a20380 x1876700049017344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763627.523976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049017728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763627.524004:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049017344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763627.524004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763627.524024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763627.524033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763627.524040:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763627.524049:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763627.524051:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049017344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763627.524065:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3077 00000100:00080000:3.0:1789763627.524104:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763627.524113:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763627.524122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763627.524139:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049017472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763627.524148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049017856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763627.524202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763627.524222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763627/real 1789763627] req@ffff9811e1a20e00 x1876700049017472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763627.524240:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049017472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763627.524251:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763627.524255:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763627.524260:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049017472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763627.524267:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763627.524288:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763627.524300:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763627/real 1789763627] req@ffff9811e1a20000 x1876700049017600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763627.524314:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049017600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763627.524323:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763627.524327:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763627.524331:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763627.524337:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763627.524356:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763627.524367:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763627/real 1789763627] req@ffff9811e1a21f80 x1876700049017728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763627.524394:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049017728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763627.524404:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763627.524408:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763627.524412:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763627.524433:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763627.524534:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763627.524550:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763627/real 1789763627] req@ffff9811e1a20a80 x1876700049017856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763627.524565:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049017856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763627.524576:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763627.524580:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763627.524584:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049017856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763627.779572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763628.419653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763628.803655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763629.443734:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763629.827671:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763630.467631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763630.851570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763631.491657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763631.875600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763632.515641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763632.643569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763632.643580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763632.643585:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763632.643598:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3082 00000100:00080000:3.0:1789763632.643612:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763632.643621:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763632.643667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049017984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763632.643702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763632.643707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763632.643710:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763632.643715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3082 00000100:00080000:3.0:1789763632.643719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763632.643723:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763632.643734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763632.643750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049018112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763632.643781:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049017984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763632.643806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763632.643815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763632.643819:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763632.643830:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3082 00000100:00080000:3.0:1789763632.643839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763632.643849:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763632.643860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763632.643887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049018240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763632.643899:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763632.643922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763632.643931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763632.643936:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763632.643946:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3082 00000100:00080000:3.0:1789763632.643954:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763632.643966:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763632/real 1789763632] req@ffff9811e1a20700 x1876700049017984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763632.643969:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763632.643991:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763632.644007:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049017984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763632.644025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049018368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763632.644113:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763632.644119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763632.644122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763632.644136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049017984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763632.644139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763632.644150:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763632.644163:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3082 00000100:00100000:0.0:1789763632.644180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049018112:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763632.644193:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763632.644209:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763632.644223:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763632.644228:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763632.644252:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763632/real 1789763632] req@ffff9811e1a21500 x1876700049018112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763632.644263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049018496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763632.644279:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049018112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763632.644309:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763632.644315:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763632.644322:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049018112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763632.644349:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763632.644375:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763632.644389:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763632/real 1789763632] req@ffff9811e1a23480 x1876700049018240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763632.644404:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049018240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763632.644414:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763632.644419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763632.644423:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763632.644441:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763632.644528:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763632.644554:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763632/real 1789763632] req@ffff9811e1a23800 x1876700049018368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763632.644582:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049018368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763632.644597:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763632.644601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763632.644605:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763632.644629:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763632.644658:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763632.644672:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763632/real 1789763632] req@ffff9811e1a23100 x1876700049018496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763632.644687:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049018496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763632.644697:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763632.644700:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763632.644704:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763632.899557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763633.539667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763633.923581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763634.563643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763634.947592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763635.587706:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763635.971619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763636.611687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763636.995577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763637.635660:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763637.763665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763637.763681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763637.763687:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763637.763703:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3087 00000100:00080000:1.0:1789763637.763734:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763637.763750:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763637.763826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700049018624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763637.763879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763637.763885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763637.763888:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763637.763894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3087 00000100:00080000:1.0:1789763637.763900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763637.763906:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763637.763922:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763637.763941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049018752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763637.763958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763637.763968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763637.763969:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049018624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763637.763974:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763637.763990:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3087 00000100:00080000:1.0:1789763637.763995:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763637.764028:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763637.764038:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763637.764091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049018880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763637.764115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763637.764117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763637.764128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763637.764133:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763637.764142:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3087 00000100:00080000:1.0:1789763637.764151:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763637.764157:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763637.764166:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763637.764187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049019008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763637.764200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763637/real 1789763637] req@ffff9811c92c7100 x1876700049018624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763637.764206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763637.764211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763637.764217:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763637.764228:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3087 00000100:00080000:1.0:1789763637.764238:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763637.764249:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763637.764256:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763637.764272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700049018624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763637.764275:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049019136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763637.764340:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763637.764350:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763637.764356:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049018624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763637.764411:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049018752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763637.764446:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763637.764521:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763637/real 1789763637] req@ffff9811c92c7800 x1876700049018752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763637.764540:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049018752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763637.764552:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763637.764556:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763637.764561:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049018752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763637.764569:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763637.764597:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763637.764611:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763637/real 1789763637] req@ffff9811c92c4700 x1876700049018880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763637.764626:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049018880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763637.764667:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763637.764672:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763637.764677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049018880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763637.764685:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763637.764716:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763637.764729:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763637/real 1789763637] req@ffff9811c92c6300 x1876700049019008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763637.764744:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049019008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763637.764754:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763637.764758:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763637.764762:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763637.764769:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763637.764788:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763637.764811:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763637/real 1789763637] req@ffff9811c92c6a00 x1876700049019136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763637.764825:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049019136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763637.764834:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763637.764838:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763637.764852:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763638.019590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763638.659697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763639.043604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763639.683667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763640.067645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763640.707716:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763641.091575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763641.731669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763642.115588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763642.755673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763642.883652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763642.883666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763642.883673:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763642.883689:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3092 00000100:00080000:3.0:1789763642.883707:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763642.883718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763642.883779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049019264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763642.883823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763642.883829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763642.883832:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763642.883837:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3092 00000100:00080000:3.0:1789763642.883841:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763642.883845:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763642.883860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763642.883876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049019392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763642.883892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763642.883926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763642.883930:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763642.883942:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3092 00000100:00080000:3.0:1789763642.883954:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763642.883956:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049019264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763642.883994:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763642.884004:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763642.884024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049019520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763642.884042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763642.884046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763642.884050:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763642.884056:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3092 00000100:00080000:3.0:1789763642.884061:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763642.884066:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763642.884073:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763642.884088:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049019648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763642.884112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763642.884114:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763642.884125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763642.884133:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763642.884142:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3092 00000100:00080000:3.0:1789763642.884148:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763642.884156:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763642.884175:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763642.884194:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763642/real 1789763642] req@ffff9811e1a22680 x1876700049019264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763642.884204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049019776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763642.884310:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049019264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763642.884327:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763642.884334:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763642.884340:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049019264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763642.884351:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049019392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763642.884386:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763642.884403:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763642/real 1789763642] req@ffff9811e1a21c00 x1876700049019392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763642.884428:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049019392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763642.884444:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763642.884451:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763642.884458:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049019392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763642.884600:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763642.884635:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763642.884676:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763642/real 1789763642] req@ffff9811e1a21880 x1876700049019520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763642.884702:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049019520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763642.884718:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763642.884725:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763642.884732:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763642.884746:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763642.884790:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763642.884809:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763642/real 1789763642] req@ffff9811e1a22300 x1876700049019648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763642.884826:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049019648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763642.884837:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763642.884841:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763642.884845:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763642.884852:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763642.884876:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763642.884887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763642/real 1789763642] req@ffff9811e1a21180 x1876700049019776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763642.884901:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049019776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763642.884921:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763642.884926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763642.884931:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049019776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763643.139608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763643.779716:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763644.163605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763644.803703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763645.187640:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763645.827672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763646.211601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763646.851656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763647.235625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763647.875702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763648.003656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763648.003670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763648.003676:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763648.003691:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3097 00000100:00080000:3.0:1789763648.003708:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763648.003719:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763648.003781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049019904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763648.003822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763648.003827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763648.003829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763648.003834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3097 00000100:00080000:3.0:1789763648.003838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763648.003842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763648.003872:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763648.003886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049020032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763648.003898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763648.003902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763648.003904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763648.003907:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3097 00000100:00100000:0.0:1789763648.003938:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049019904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763648.003953:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763648.003960:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763648.003966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763648.003983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049020160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763648.003997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763648.004003:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763648.004006:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763648.004012:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3097 00000100:00080000:3.0:1789763648.004019:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763648.004025:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763648.004033:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763648.004054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049020288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763648.004055:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763648.004071:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763648.004075:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763648.004078:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763648.004083:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3097 00000100:00080000:3.0:1789763648.004089:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763648.004093:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763648.004098:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763648.004112:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049020416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763648.004126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763648/real 1789763648] req@ffff9811e1a22a00 x1876700049019904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763648.004179:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049019904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763648.004198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763648.004205:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763648.004212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049019904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763648.004224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049020032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763648.004265:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763648.004279:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763648/real 1789763648] req@ffff9811e1a22d80 x1876700049020032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763648.004291:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049020032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763648.004299:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763648.004302:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763648.004306:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049020032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763648.004345:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763648.004365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763648.004375:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763648/real 1789763648] req@ffff9811e1a21180 x1876700049020160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763648.004387:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049020160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763648.004395:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763648.004398:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763648.004401:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763648.004405:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763648.004420:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763648.004427:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763648/real 1789763648] req@ffff9811e1a22300 x1876700049020288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763648.004437:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049020288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763648.004445:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763648.004447:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763648.004450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763648.004454:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763648.004503:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763648.004513:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763648/real 1789763648] req@ffff9811e1a21880 x1876700049020416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763648.004534:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049020416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763648.004542:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763648.004545:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763648.004547:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763648.259628:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763648.899657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763649.283619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763649.923608:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763650.307545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763650.947687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763651.331619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763651.971610:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789763652.355578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763652.995611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763653.123611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763653.123620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763653.123624:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763653.123648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3102 00000100:00080000:1.0:1789763653.123658:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763653.123665:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763653.123695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049020544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763653.123719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763653.123722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763653.123723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763653.123726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3102 00000100:00080000:1.0:1789763653.123728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763653.123730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763653.123738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763653.123748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049020672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763653.123760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763653.123763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763653.123764:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763653.123768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3102 00000100:00080000:1.0:1789763653.123770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763653.123772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763653.123775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763653.123783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049020800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763653.123806:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049020544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763653.123811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763653.123824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763653.123828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763653.123834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3102 00000100:00080000:1.0:1789763653.123837:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763653.123840:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763653.123846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763653.123866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049020928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763653.123872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763653.123875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763653.123877:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763653.123880:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3102 00000100:00080000:1.0:1789763653.123893:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763653.123897:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763653.123901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763653.123908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049021056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763653.123964:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763653.124029:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763653/real 1789763653] req@ffff9811c92c5f80 x1876700049020544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763653.124058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049020544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763653.124073:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763653.124079:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763653.124085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049020544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763653.124095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049020672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763653.124123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763653.124138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763653/real 1789763653] req@ffff9811c92c4a80 x1876700049020672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763653.124153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049020672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763653.124188:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763653.124193:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763653.124197:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049020672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763653.124234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763653.124268:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763653.124290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763653/real 1789763653] req@ffff9811c92c4380 x1876700049020800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763653.124316:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049020800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763653.124333:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763653.124340:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763653.124346:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763653.124359:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763653.124391:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763653.124406:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763653/real 1789763653] req@ffff9811c92c6680 x1876700049020928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763653.124422:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049020928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763653.124432:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763653.124436:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763653.124450:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049020928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763653.124508:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763653.124557:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763653.124579:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763653/real 1789763653] req@ffff9811c92c5c00 x1876700049021056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763653.124596:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049021056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763653.124607:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763653.124611:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763653.124615:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763653.379754:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763654.019742:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763654.403581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763655.043551:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763655.427644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763656.067658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763656.451592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763657.091658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763657.475609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763658.115638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763658.243592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763658.243618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763658.243625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763658.243641:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3107 00000100:00080000:1.0:1789763658.243685:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763658.243698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763658.243755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049021184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763658.243817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763658.243825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763658.243829:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763658.243837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3107 00000100:00080000:1.0:1789763658.243869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763658.243880:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763658.243899:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763658.243918:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049021184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763658.243929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049021312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763658.243951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763658.243957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763658.243960:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763658.243965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3107 00000100:00080000:1.0:1789763658.243972:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763658.243978:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763658.243985:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763658.244000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049021440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763658.244014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763658.244029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763658.244032:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763658.244038:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3107 00000100:00080000:1.0:1789763658.244045:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763658.244051:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763658.244059:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763658.244080:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049021568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763658.244097:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763658.244099:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049021312:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763658.244109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763658.244116:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763658.244122:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3107 00000100:00080000:1.0:1789763658.244129:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763658.244134:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763658.244147:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763658.244163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763658.244180:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049021696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763658.244207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763658.244252:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763658.499581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763659.139700:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763659.523549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763660.163724:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763660.547572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763661.187693:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763661.315951:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763661.316009:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763661.316030:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763661.316098:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763661.316108:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763661.316119:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763661.316130:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763661.316203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763658/real 1789763661] req@ffff9811c92c6d80 x1876700049021696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763661.316245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049021696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763661.316294:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763661.316302:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763661.316310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763661.316323:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763658/real 1789763661] req@ffff9811c92c7480 x1876700049021568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763661.316356:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049021568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763661.316370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763661.316375:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763661.316381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763661.316391:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763658/real 1789763661] req@ffff9811c92c4000 x1876700049021312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763661.316408:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049021312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763661.316429:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763661.316435:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763661.316440:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049021312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763661.316450:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763658/real 1789763661] req@ffff9811c92c5500 x1876700049021440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763661.316522:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049021440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763661.316546:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763661.316554:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763661.316561:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049021440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763661.316573:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763658/real 1789763661] req@ffff9811c92c4e00 x1876700049021184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763661.316593:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049021184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763661.316605:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763661.316609:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763661.316614:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049021184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763661.571600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763662.211709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763662.595551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763663.235742:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763663.363681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763663.363697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763663.363704:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763663.363720:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3112 00000100:00080000:1.0:1789763663.363764:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763663.363777:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763663.363929:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049021824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763663.363993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763663.364002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763663.364007:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763663.364018:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3112 00000100:00080000:0.0:1789763663.364033:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763663.364045:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763663.364077:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763663.364115:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049021952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763663.364158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049021824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763663.364183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763663.364195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763663.364200:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763663.364214:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3112 00000100:00080000:0.0:1789763663.364226:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763663.364238:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763663.364253:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763663.364288:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049022080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763663.364311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763663.364317:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763663.364325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763663.364334:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763663.364346:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3112 00000100:00080000:0.0:1789763663.364357:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763663.364368:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763663.364381:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763663.364409:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049022208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763663.364429:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763663/real 1789763663] req@ffff9811c92c5180 x1876700049021824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763663.364435:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763663.364557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763663.364563:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763663.364577:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3112 00000100:00100000:2.0:1789763663.364577:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049021824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763663.364610:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763663.364611:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763663.364624:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763663.364644:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1789763663.364647:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763663.364664:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049021824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763663.364696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049022336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763663.364734:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049021952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763663.364832:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763663.364871:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763663/real 1789763663] req@ffff9811c2e2a680 x1876700049021952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763663.364917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049021952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763663.364942:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763663.364950:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763663.364962:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049021952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763663.364993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763663.365060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763663.365099:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763663/real 1789763663] req@ffff9811c2e2bb80 x1876700049022080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763663.365136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049022080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763663.365156:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763663.365166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763663.365175:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763663.365193:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763663.365264:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763663.365297:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763663/real 1789763663] req@ffff9811c2e29c00 x1876700049022208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763663.365320:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049022208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763663.365335:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763663.365341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763663.365348:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763663.365384:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763663.365424:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763663.365448:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763663/real 1789763663] req@ffff9811c2e29880 x1876700049022336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763663.365526:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049022336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763663.365544:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763663.365552:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763663.365560:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763663.619608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763664.259709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763664.643594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763665.283715:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763665.667540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763666.307647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763666.691538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763667.331743:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763667.715612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763668.355636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763668.483614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763668.483622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763668.483644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763668.483668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3118 00000100:00080000:2.0:1789763668.483679:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763668.483685:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763668.483718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049022464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763668.483747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763668.483751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763668.483752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763668.483756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3118 00000100:00080000:2.0:1789763668.483759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763668.483764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763668.483788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763668.483801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049022464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763668.483809:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049022592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763668.483827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763668.483832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763668.483835:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763668.483840:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3118 00000100:00080000:2.0:1789763668.483845:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763668.483850:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763668.483856:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763668.483870:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763668.483871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049022720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763668.483894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763668.483897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763668.483899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763668.483903:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3118 00000100:00080000:2.0:1789763668.483908:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763668.483911:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763668.483916:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763668/real 1789763668] req@ffff9811d8639500 x1876700049022464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763668.483919:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763668.483953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049022848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763668.483961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049022464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763668.483966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763668.483975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763668.483979:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763668.483985:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3118 00000100:00080000:0.0:1789763668.483987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763668.483989:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763668.483994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763668.483997:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763668.484003:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049022464:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763668.484007:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763668.484025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049022976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763668.484030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049022592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763668.484061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763668.484080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763668/real 1789763668] req@ffff9811d8639880 x1876700049022592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763668.484097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049022592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763668.484108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763668.484112:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763668.484142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049022592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763668.484149:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763668.484165:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763668.484174:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763668/real 1789763668] req@ffff9811d863b100 x1876700049022720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763668.484182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049022720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763668.484187:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763668.484189:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763668.484191:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763668.484195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763668.484205:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763668.484211:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763668/real 1789763668] req@ffff9811d8639f80 x1876700049022848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763668.484218:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049022848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763668.484223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763668.484225:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763668.484227:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763668.484238:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763668.484248:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763668.484261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763668/real 1789763668] req@ffff9811d863ad80 x1876700049022976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763668.484269:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049022976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763668.484274:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763668.484276:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763668.484278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049022976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763668.739543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763669.379693:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763669.763540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763670.403627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763670.787575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763671.427705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763671.811594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763672.451618:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763672.835578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763673.475778:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763673.603680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763673.603691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763673.603697:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763673.603710:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3123 00000100:00080000:1.0:1789763673.603724:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763673.603758:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763673.603808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049023104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763673.603850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763673.603855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763673.603857:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763673.603862:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3123 00000100:00080000:1.0:1789763673.603867:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763673.603871:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763673.603883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763673.603897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049023232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763673.603912:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763673.603936:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763673.603940:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763673.603946:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3123 00000100:00080000:1.0:1789763673.603953:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763673.603956:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049023104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763673.603965:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763673.603976:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763673.603996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049023360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763673.604012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763673.604016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763673.604019:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763673.604024:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3123 00000100:00080000:1.0:1789763673.604042:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763673.604047:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763673.604054:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763673.604057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763673.604074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049023488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763673.604089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763673.604093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763673.604095:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763673.604100:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3123 00000100:00080000:1.0:1789763673.604106:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763673.604111:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763673.604123:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763673/real 1789763673] req@ffff9811c92c7b80 x1876700049023104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789763673.604130:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763673.604156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049023616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763673.604165:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049023104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763673.604191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763673.604197:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763673.604203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049023104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763673.604214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049023232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763673.604242:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763673.604256:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763673/real 1789763673] req@ffff9811c92c5880 x1876700049023232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763673.604271:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049023232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763673.604304:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763673.604309:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763673.604313:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049023232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763673.604351:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763673.604393:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763673.604417:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763673/real 1789763673] req@ffff9811c92c5180 x1876700049023360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763673.604444:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049023360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763673.604512:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763673.604518:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763673.604523:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763673.604532:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763673.604564:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763673.604580:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763673/real 1789763673] req@ffff9811c92c4e00 x1876700049023488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763673.604596:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049023488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763673.604605:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763673.604609:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763673.604624:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763673.604632:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763673.604658:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763673.604671:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763673/real 1789763673] req@ffff9811c92c5500 x1876700049023616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763673.604685:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049023616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763673.604695:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763673.604698:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763673.604702:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049023616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763673.859572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763674.499666:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763674.883582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763675.523716:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763675.907602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763676.547668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763676.931613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763677.571723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763677.955561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763678.595568:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763678.723544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763678.723552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763678.723556:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763678.723565:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3128 00000100:00080000:1.0:1789763678.723573:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763678.723580:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763678.723612:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049023744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763678.723656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763678.723661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763678.723664:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763678.723669:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3128 00000100:00080000:1.0:1789763678.723673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763678.723678:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763678.723691:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763678.723748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049023872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763678.723763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763678.723768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763678.723776:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049023744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763678.723783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763678.723803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3128 00000100:00080000:1.0:1789763678.723807:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763678.723811:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763678.723816:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763678.723833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049024000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763678.723840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763678.723843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763678.723844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763678.723847:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3128 00000100:00080000:1.0:1789763678.723850:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763678.723853:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763678.723857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763678.723867:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049024128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763678.723873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763678.723875:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763678.723876:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763678.723879:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3128 00000100:00080000:1.0:1789763678.723889:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763678.723893:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763678.723896:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763678.723904:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763678.723921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049024256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763678.724014:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763678/real 1789763678] req@ffff9811c92c4000 x1876700049023744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763678.724047:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049023744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763678.724064:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763678.724070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763678.724085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049023744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763678.724125:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049023872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763678.724160:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763678.724200:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763678/real 1789763678] req@ffff9811c92c7480 x1876700049023872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763678.724218:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049023872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763678.724230:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763678.724234:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763678.724239:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049023872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763678.724247:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763678.724273:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763678.724286:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763678/real 1789763678] req@ffff9811c92c6d80 x1876700049024000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763678.724310:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049024000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763678.724319:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763678.724323:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763678.724326:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763678.724333:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763678.724353:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763678.724377:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763678/real 1789763678] req@ffff9811c92c5c00 x1876700049024128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763678.724392:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049024128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763678.724413:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763678.724417:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763678.724422:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763678.724430:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763678.724457:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763678.724516:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763678/real 1789763678] req@ffff9811c92c6680 x1876700049024256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763678.724532:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049024256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763678.724544:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763678.724551:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763678.724558:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763678.979555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763679.619752:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763680.003586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763680.643670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763681.027596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763681.667790:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763682.051577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763682.691647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763683.075541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763683.715597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763683.843562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763683.843571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763683.843575:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763683.843583:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3133 00000100:00080000:2.0:1789763683.843592:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763683.843604:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763683.843636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700049024384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763683.843656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763683.843670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763683.843672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763683.843675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3133 00000100:00080000:2.0:1789763683.843678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763683.843681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763683.843690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763683.843701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700049024512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763683.843706:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049024384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763683.843718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763683.843721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763683.843723:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763683.843726:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3133 00000100:00080000:2.0:1789763683.843728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763683.843731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763683.843746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763683.843759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700049024640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763683.843769:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763683.843775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763683.843780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763683.843782:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763683.843789:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3133 00000100:00080000:2.0:1789763683.843793:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763683.843799:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763683.843804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763683.843808:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763683/real 1789763683] req@ffff9811d8639180 x1876700049024384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763683.843820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049024768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763683.843827:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700049024384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763683.843830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763683.843838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763683.843841:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763683.843846:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763683.843846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3133 00000100:00080000:2.0:1789763683.843854:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763683.843856:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:2.0:1789763683.843860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763683.843864:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049024384:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763683.843866:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763683.843876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049024896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763683.843897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049024512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763683.843918:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763683.843927:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763683/real 1789763683] req@ffff9811d863bb80 x1876700049024512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763683.843935:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700049024512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763683.843941:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763683.843943:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763683.843945:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049024512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763683.843949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763683.843962:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763683.843973:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763683/real 1789763683] req@ffff9811d8639c00 x1876700049024640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763683.843986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700049024640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763683.843994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763683.843997:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763683.844001:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763683.844008:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763683.844029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763683.844041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763683/real 1789763683] req@ffff9811d863ad80 x1876700049024768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763683.844063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049024768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763683.844070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763683.844072:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763683.844074:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763683.844079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763683.844095:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763683.844102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763683/real 1789763683] req@ffff9811d8639f80 x1876700049024896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763683.844110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049024896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763683.844115:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763683.844117:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763683.844119:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049024896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763684.099636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763684.739677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763685.123597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763685.763697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763686.147548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763686.787622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763687.171591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763687.811803:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763688.195575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763688.835646:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763688.963562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763688.963574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763688.963579:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763688.963593:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3138 00000100:00080000:2.0:1789763688.963606:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763688.963615:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763688.963688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049025024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763688.963726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763688.963731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763688.963734:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763688.963739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3138 00000100:00080000:2.0:1789763688.963744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763688.963748:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763688.963760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763688.963778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049025152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763688.963795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763688.963800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763688.963804:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763688.963808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3138 00000100:00080000:2.0:1789763688.963811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763688.963817:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763688.963842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763688.963860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049025280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763688.963873:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049025024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763688.963880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763688.963898:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763688.963903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763688.963910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3138 00000100:00080000:2.0:1789763688.963915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763688.963920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763688.963927:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763688.963942:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049025408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763688.963954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763688.963959:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763688.963962:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763688.963966:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3138 00000100:00080000:2.0:1789763688.963973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763688.963979:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763688.963985:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763688.964001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049025536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763688.964034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763688.964120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763688/real 1789763688] req@ffff9811d863b100 x1876700049025024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763688.964150:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049025024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763688.964197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763688.964204:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763688.964210:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049025024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763688.964224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049025152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763688.964274:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763688.964328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763688/real 1789763688] req@ffff9811d8639880 x1876700049025152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763688.964351:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049025152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763688.964370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763688.964375:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763688.964379:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049025152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763688.964422:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763688.964452:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763688.964537:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763688/real 1789763688] req@ffff9811d8639500 x1876700049025280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763688.964554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049025280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763688.964565:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763688.964569:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763688.964573:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763688.964593:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763688.964654:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763688.964674:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763688/real 1789763688] req@ffff9811d863a680 x1876700049025408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763688.964690:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049025408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763688.964701:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763688.964706:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763688.964710:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763688.964718:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763688.964743:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763688.964755:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763688/real 1789763688] req@ffff9811d8638a80 x1876700049025536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763688.964769:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049025536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763688.964778:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763688.964782:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763688.964786:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763689.219615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763689.859701:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763690.243551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763690.883603:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763691.267555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763691.907730:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763692.291639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763692.931730:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763693.315578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763693.955699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763694.083644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763694.083686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763694.083693:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763694.083711:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3143 00000100:00080000:2.0:1789763694.083730:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763694.083744:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763694.083807:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049025664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763694.083848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763694.083853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763694.083857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763694.083863:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3143 00000100:00080000:2.0:1789763694.083868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763694.083874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763694.083890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763694.083908:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049025664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763694.083914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049025792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763694.083940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763694.083946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763694.083949:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763694.083956:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3143 00000100:00080000:2.0:1789763694.083964:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763694.083970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763694.083977:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763694.083996:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049025920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763694.084021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763694.084025:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763694.084028:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763694.084034:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3143 00000800:00000100:0.0:1789763694.084044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763694.084045:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763694.084058:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763694.084069:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763694.084095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049026048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763694.084115:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763694.084123:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763694.084128:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763694.084138:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3143 00000100:00080000:2.0:1789763694.084148:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763694.084151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763694/real 1789763694] req@ffff9811d863b800 x1876700049025664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763694.084162:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763694.084183:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763694.084208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049025664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763694.084216:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049026176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763694.084265:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763694.084275:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763694.084282:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049025664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763694.084325:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049025792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763694.084363:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763694.084385:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763694/real 1789763694] req@ffff9811d863a300 x1876700049025792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763694.084402:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049025792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763694.084414:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763694.084419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763694.084424:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049025792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763694.084433:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763694.084508:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763694.084539:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763694/real 1789763694] req@ffff9811d863b480 x1876700049025920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763694.084559:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049025920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763694.084622:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763694.084628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763694.084636:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049025920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763694.084647:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763694.084681:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763694.084697:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763694/real 1789763694] req@ffff9811d863aa00 x1876700049026048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763694.084712:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049026048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763694.084722:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763694.084727:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763694.084730:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763694.084737:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763694.084756:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763694.084767:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763694/real 1789763694] req@ffff9811d8638000 x1876700049026176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763694.084781:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049026176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763694.084791:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763694.084794:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763694.084808:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763694.339622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763694.979713:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763695.363569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763696.003657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763696.387623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763697.027700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763697.411594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763698.051729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763698.435593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763699.075745:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763699.203614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763699.203631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763699.203638:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763699.203653:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3148 00000100:00080000:2.0:1789763699.203695:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763699.203710:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763699.203765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700049026304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763699.203807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763699.203812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763699.203814:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763699.203820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3148 00000100:00080000:2.0:1789763699.203825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763699.203829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763699.203842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763699.203858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700049026432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763699.203875:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763699.203880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763699.203884:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763699.203889:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3148 00000100:00080000:2.0:1789763699.203894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763699.203903:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763699.203905:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049026304:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763699.203910:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763699.203927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700049026560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763699.203941:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763699.203956:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763699.203959:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763699.203964:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3148 00000100:00080000:2.0:1789763699.203972:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763699.203977:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763699.203984:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763699.204009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049026688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763699.204010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763699.204030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763699.204035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763699.204038:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763699.204043:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3148 00000100:00080000:2.0:1789763699.204049:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763699.204068:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763699.204074:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763699.204101:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763699/real 1789763699] req@ffff9811d8638380 x1876700049026304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763699.204102:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049026816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763699.204168:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700049026304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763699.204185:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763699.204191:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763699.204196:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049026304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763699.204207:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049026432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763699.204240:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763699.204257:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763699/real 1789763699] req@ffff9811d8638e00 x1876700049026432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763699.204272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700049026432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763699.204283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763699.204287:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763699.204291:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049026432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763699.204320:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763699.204341:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763699.204370:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763699/real 1789763699] req@ffff9811d8638700 x1876700049026560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763699.204386:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700049026560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763699.204396:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763699.204400:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763699.204404:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763699.204411:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763699.204433:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763699.204444:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763699/real 1789763699] req@ffff9811d8638000 x1876700049026688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763699.204502:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049026688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763699.204513:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763699.204517:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763699.204521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763699.204529:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763699.204590:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763699.204614:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763699/real 1789763699] req@ffff9811d863aa00 x1876700049026816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763699.204640:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049026816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763699.204670:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763699.204675:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763699.204679:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049026816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763699.459629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763700.099648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763700.483624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763701.123808:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763701.507593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763702.147733:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763702.531582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763703.171749:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763703.555648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763704.195644:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763704.323597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763704.323620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763704.323625:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763704.323636:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3153 00000100:00080000:2.0:1789763704.323646:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763704.323655:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763704.323693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049026944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763704.323750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763704.323754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763704.323757:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763704.323762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3153 00000100:00080000:2.0:1789763704.323767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763704.323772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763704.323782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763704.323798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049027072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763704.323812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763704.323818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763704.323821:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763704.323826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3153 00000100:00080000:2.0:1789763704.323830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763704.323836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763704.323842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763704.323863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049027200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763704.323864:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049026944:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763704.323896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763704.323901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763704.323904:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763704.323910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3153 00000100:00080000:2.0:1789763704.323926:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763704.323942:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763704.323949:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763704.323966:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049027328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763704.323977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763704.323981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763704.323983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763704.323987:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3153 00000800:00000100:0.0:1789763704.323995:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763704.323995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763704.324007:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763704.324014:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763704.324041:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049027456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763704.324059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763704/real 1789763704] req@ffff9811d863b480 x1876700049026944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763704.324083:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049026944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763704.324094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763704.324098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763704.324102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049026944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763704.324110:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049027072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763704.324132:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763704.324142:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763704/real 1789763704] req@ffff9811d863a300 x1876700049027072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763704.324151:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049027072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763704.324157:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763704.324160:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763704.324162:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049027072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763704.324199:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763704.324217:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763704.324225:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763704/real 1789763704] req@ffff9811d863b800 x1876700049027200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763704.324235:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049027200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763704.324241:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763704.324244:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763704.324246:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763704.324250:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763704.324262:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763704.324268:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763704/real 1789763704] req@ffff9811d8638a80 x1876700049027328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763704.324276:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049027328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763704.324282:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763704.324284:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763704.324286:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763704.324290:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763704.324301:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763704.324316:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763704/real 1789763704] req@ffff9811d863a680 x1876700049027456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763704.324333:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049027456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763704.324340:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763704.324342:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763704.324344:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763704.579635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763705.219765:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763705.603635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763706.243734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763706.627558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763707.267663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763707.651570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763708.291675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763708.675594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763709.315669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763709.443580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763709.443595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763709.443601:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763709.443619:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3159 00000100:00080000:2.0:1789763709.443637:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763709.443650:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763709.443719:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049027584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763709.443760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763709.443794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763709.443798:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763709.443807:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3159 00000100:00080000:2.0:1789763709.443814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763709.443820:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763709.443837:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763709.443846:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049027584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763709.443863:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049027712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763709.443894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763709.443901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763709.443905:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763709.443913:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3159 00000100:00080000:2.0:1789763709.443922:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763709.443931:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763709.443943:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763709.443972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049027840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763709.443979:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763709.443999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763709.444006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763709.444009:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763709.444018:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3159 00000100:00080000:2.0:1789763709.444025:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763709.444030:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763709.444052:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763709.444065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763709/real 1789763709] req@ffff9811d8639500 x1876700049027584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763709.444076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049027968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763709.444107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763709.444112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763709.444115:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763709.444120:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049027584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763709.444126:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3159 00000100:00080000:0.0:1789763709.444155:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763709.444155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763709.444168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763709.444177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:2.0:1789763709.444181:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763709.444195:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049027584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763709.444203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049028096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763709.444259:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049027712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763709.444301:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763709.444320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763709/real 1789763709] req@ffff9811d8639880 x1876700049027712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763709.444337:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049027712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763709.444369:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763709.444374:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763709.444379:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049027712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763709.444388:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763709.444430:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763709.444454:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763709/real 1789763709] req@ffff9811d863b100 x1876700049027840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763709.444562:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049027840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763709.444574:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763709.444578:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763709.444583:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763709.444592:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763709.444625:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763709.444641:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763709/real 1789763709] req@ffff9811d8639f80 x1876700049027968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763709.444656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049027968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763709.444666:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763709.444670:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763709.444686:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049027968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763709.444694:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763709.444721:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763709.444734:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763709/real 1789763709] req@ffff9811d863ad80 x1876700049028096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763709.444749:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049028096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763709.444759:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763709.444762:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763709.444766:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763709.699641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763710.339599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763710.723656:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763711.363572:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763711.747604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763712.387736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763712.771603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763713.411672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763713.795602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763714.435702:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763714.563636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763714.563649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763714.563655:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763714.563668:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3164 00000100:00080000:1.0:1789763714.563682:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763714.563691:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763714.563750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049028224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763714.563788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763714.563793:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763714.563795:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763714.563800:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3164 00000100:00080000:1.0:1789763714.563805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763714.563810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763714.563823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763714.563861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049028352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763714.563881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763714.563883:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049028224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763714.563887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763714.563896:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763714.563909:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3164 00000100:00080000:1.0:1789763714.563914:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763714.563921:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763714.563929:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763714.563945:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049028480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763714.563959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763714.563963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763714.563965:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763714.563969:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3164 00000100:00080000:1.0:1789763714.563977:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763714.563982:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763714.563989:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763714.563991:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763714.564009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049028608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763714.564024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763714.564029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763714.564032:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763714.564045:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3164 00000100:00080000:1.0:1789763714.564051:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763714.564059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763714/real 1789763714] req@ffff9811c92c4380 x1876700049028224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763714.564060:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763714.564074:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763714.564113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049028736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763714.564130:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049028224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763714.564151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763714.564161:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763714.564168:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049028224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763714.564224:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049028352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763714.564246:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763714.564299:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763714/real 1789763714] req@ffff9811c92c4a80 x1876700049028352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763714.564316:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049028352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763714.564328:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763714.564332:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763714.564337:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049028352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763714.564345:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763714.564369:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763714.564381:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763714/real 1789763714] req@ffff9811c92c5f80 x1876700049028480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763714.564409:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049028480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763714.564414:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763714.564416:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763714.564419:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763714.564423:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763714.564453:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763714.564526:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763714/real 1789763714] req@ffff9811c92c6a00 x1876700049028608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763714.564535:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049028608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763714.564568:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763714.564573:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763714.564577:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763714.564586:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763714.564618:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763714.564632:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763714/real 1789763714] req@ffff9811c92c6300 x1876700049028736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763714.564647:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049028736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763714.564656:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763714.564660:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763714.564663:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049028736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763714.819560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763715.459746:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763715.843600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763716.483760:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763716.867567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763717.507706:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763717.891641:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763718.531675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763718.915583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763719.555797:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763719.683631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763719.683644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763719.683650:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763719.683663:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3169 00000100:00080000:2.0:1789763719.683689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763719.683699:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763719.683747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700049028864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763719.683785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763719.683790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763719.683793:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763719.683798:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3169 00000100:00080000:2.0:1789763719.683826:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763719.683837:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763719.683854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763719.683886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700049028992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763719.683924:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049028864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763719.683928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763719.683953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763719.683958:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763719.683970:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3169 00000100:00080000:2.0:1789763719.683979:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763719.683991:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763719.684004:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763719.684034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700049029120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763719.684055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763719.684060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763719.684062:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763719.684068:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3169 00000100:00080000:2.0:1789763719.684074:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763719.684086:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763719.684097:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:2.0:1789763719.684105:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763719.684134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049029248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763719.684170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763719.684176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763719.684179:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763719.684188:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763719/real 1789763719] req@ffff9811d8639c00 x1876700049028864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763719.684193:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3169 00000100:00080000:2.0:1789763719.684213:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763719.684236:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763719.684246:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763719.684259:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700049028864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763719.684267:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049029376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763719.684312:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763719.684322:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763719.684332:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049028864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763719.684426:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049028992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763719.684563:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763719.684587:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763719/real 1789763719] req@ffff9811d863bb80 x1876700049028992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763719.684606:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700049028992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763719.684618:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763719.684624:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763719.684628:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049028992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763719.684637:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763719.684664:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763719.684676:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763719/real 1789763719] req@ffff9811d8639180 x1876700049029120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763719.684691:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700049029120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763719.684700:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763719.684704:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763719.684708:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763719.684715:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763719.684735:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763719.684746:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763719/real 1789763719] req@ffff9811d863ad80 x1876700049029248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763719.684787:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049029248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763719.684804:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763719.684810:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763719.684817:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763719.684830:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763719.684876:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763719.684898:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763719/real 1789763719] req@ffff9811d8639f80 x1876700049029376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763719.684922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049029376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763719.684937:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763719.684945:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763719.684952:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763719.939599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763720.579703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763720.963544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763721.603578:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763721.987545:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763722.627649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763723.011615:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763723.651629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763724.035593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763724.675691:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763724.803631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763724.803649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763724.803656:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763724.803673:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3174 00000100:00080000:1.0:1789763724.803694:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763724.803709:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763724.803789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049029504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763724.803846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763724.803854:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763724.803859:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763724.803867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3174 00000100:00080000:1.0:1789763724.803875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763724.803883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763724.803902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763724.803932:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049029632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763724.803958:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763724.803977:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049029504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763724.803994:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763724.804015:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763724.804024:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3174 00000100:00080000:1.0:1789763724.804032:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763724.804039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763724.804049:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763724.804069:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700049029760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763724.804086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763724.804091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763724.804094:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763724.804099:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3174 00000100:00080000:1.0:1789763724.804104:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763724.804108:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763724.804116:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763724.804134:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049029888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763724.804149:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763724.804154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763724.804158:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763724.804163:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049029632:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763724.804170:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3174 00000100:00080000:1.0:1789763724.804182:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763724.804187:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763724.804206:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763724.804229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049030016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763724.804248:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763724.804286:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763724.804321:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763725.059600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763725.699670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763726.083596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763726.723736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763727.107580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763727.747619:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763727.875868:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789763727.875924:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763727.875949:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763727.876012:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763727.876031:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763727.876049:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763727.876069:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763727.876124:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763724/real 1789763727] req@ffff9811c92c6a00 x1876700049030016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763727.876182:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049030016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763727.876205:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763727.876212:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763727.876218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763727.876229:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763724/real 1789763727] req@ffff9811c92c7100 x1876700049029760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763727.876254:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700049029760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763727.876264:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763727.876269:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763727.876273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763727.876280:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763724/real 1789763727] req@ffff9811c92c6300 x1876700049029888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763727.876294:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049029888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763727.876302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763727.876306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763727.876310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049029888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763727.876422:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763724/real 1789763727] req@ffff9811c92c4700 x1876700049029504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763727.876453:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049029504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763727.876522:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763727.876532:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763727.876542:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049029504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763727.876558:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763724/real 1789763727] req@ffff9811c92c7800 x1876700049029632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763727.876589:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049029632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763727.876606:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763727.876613:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763727.876619:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049029632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763728.131603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763728.771729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763729.155559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763729.795653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763729.923588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763729.923604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763729.923611:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763729.923630:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3179 00000100:00080000:1.0:1789763729.923647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763729.923662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763729.923729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049030144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763729.923776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763729.923784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763729.923789:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763729.923798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3179 00000100:00080000:1.0:1789763729.923809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763729.923820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763729.923839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763729.923860:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049030144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763729.923877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049030272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763729.923908:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763729.923915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763729.923920:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763729.923929:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3179 00000800:00000100:3.0:1789763729.923982:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763729.923983:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763729.924002:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763729.924017:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763729.924077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763729/real 1789763729] req@ffff9811c92c5f80 x1876700049030144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763729.924078:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049030400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763729.924129:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763729.924136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763729.924143:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763729.924159:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3179 00000100:00080000:1.0:1789763729.924170:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763729.924180:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763729.924192:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763729.924194:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049030144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763729.924223:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789763729.924232:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049030528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763729.924241:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789763729.924257:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763729.924271:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1789763729.924278:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049030144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763729.924280:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763729.924298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3179 00000100:00080000:1.0:1789763729.924308:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763729.924317:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763729.924363:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763729.924374:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049030272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763729.924399:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049030656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763729.924444:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763729.924567:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763729/real 1789763729] req@ffff9811c92c4a80 x1876700049030272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763729.924599:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049030272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763729.924618:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763729.924626:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763729.924634:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049030272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763729.924648:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763729.924695:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763729.924720:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763729/real 1789763729] req@ffff9811c92c4380 x1876700049030400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763729.924750:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049030400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763729.925784:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763729.925797:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763729.925807:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763729.925826:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763729.925891:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763729.926033:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763729/real 1789763729] req@ffff9811c92c6680 x1876700049030528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763729.926062:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049030528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763729.926080:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763729.926088:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763729.926095:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763729.926146:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763729.926193:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763729.926211:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763729/real 1789763729] req@ffff9811c92c5c00 x1876700049030656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763729.926345:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049030656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763729.926365:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763729.926373:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763729.926380:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049030656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763730.179568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763730.819721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763731.203556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763731.843657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763732.227558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763732.867805:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763733.251549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763733.891657:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763734.275552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763734.915743:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763735.043716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763735.043739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763735.043746:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763735.043763:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3184 00000100:00080000:2.0:1789763735.043814:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763735.043826:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763735.043886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049030784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763735.043944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763735.043950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763735.043953:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763735.043958:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3184 00000100:00080000:2.0:1789763735.043963:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763735.043967:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763735.043983:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763735.044001:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049030912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763735.044017:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763735.044023:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763735.044026:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763735.044031:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3184 00000100:00080000:2.0:1789763735.044036:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763735.044052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763735.044054:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049030784:192.168.203.113@tcp:250:kworker.0 00000008:00100000:2.0:1789763735.044063:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763735.044081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049031040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763735.044119:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763735.044139:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763735.044144:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763735.044152:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3184 00000100:00080000:2.0:1789763735.044159:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763735.044167:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763735.044176:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763735.044206:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763735.044210:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049031168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763735.044228:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763735.044233:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763735.044235:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763735.044242:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3184 00000100:00080000:2.0:1789763735.044247:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763735.044253:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763735.044260:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763735.044276:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049031296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763735.044314:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763735/real 1789763735] req@ffff9811d863b100 x1876700049030784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763735.044364:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049030784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763735.044383:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763735.044389:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763735.044395:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049030784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763735.044406:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049030912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763735.044443:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763735.044510:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763735/real 1789763735] req@ffff9811d8639880 x1876700049030912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763735.044530:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049030912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763735.044541:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763735.044545:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763735.044549:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049030912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763735.044581:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763735.044606:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763735.044620:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763735/real 1789763735] req@ffff9811d8639500 x1876700049031040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763735.044658:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049031040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763735.044669:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763735.044673:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763735.044677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763735.044684:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763735.044709:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763735.044722:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763735/real 1789763735] req@ffff9811d863a680 x1876700049031168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763735.044736:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049031168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763735.044745:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763735.044749:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763735.044752:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763735.044759:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763735.044778:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763735.044791:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763735/real 1789763735] req@ffff9811d8638a80 x1876700049031296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763735.044814:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049031296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763735.044881:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763735.044890:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763735.044897:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763735.299517:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763735.939631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763736.323557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763736.963729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763737.347557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763737.987662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763738.371516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763739.011672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763739.395559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763740.035965:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763740.163652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763740.163666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763740.163672:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763740.163684:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3189 00000100:00080000:3.0:1789763740.163711:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763740.163722:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763740.163771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049031424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763740.163852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763740.163857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763740.163860:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763740.163865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3189 00000100:00080000:3.0:1789763740.163870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763740.163874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763740.163888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763740.163916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049031552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763740.163931:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049031424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763740.163970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763740.163979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763740.163983:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763740.163994:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3189 00000100:00080000:3.0:1789763740.164004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763740.164011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763740.164020:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763740.164026:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763740.164048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049031680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763740.164063:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763740.164068:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763740.164071:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763740.164076:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3189 00000100:00080000:3.0:1789763740.164081:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763740.164087:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763740.164094:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763740/real 1789763740] req@ffff9811e1a21c00 x1876700049031424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763740.164098:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763740.164121:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049031808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763740.164136:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049031424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763740.164138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763740.164154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763740.164160:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763740.164174:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3189 00000100:00080000:0.0:1789763740.164182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763740.164190:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763740.164197:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763740.164204:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763740.164214:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763740.164230:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049031936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763740.164233:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049031424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763740.164328:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049031552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763740.164369:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763740.164389:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763740/real 1789763740] req@ffff9811e1a22680 x1876700049031552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763740.164408:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049031552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763740.164420:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763740.164424:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763740.164429:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049031552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763740.164437:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763740.164458:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763740.164520:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763740/real 1789763740] req@ffff9811e1a23100 x1876700049031680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763740.164535:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049031680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763740.164546:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763740.164550:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763740.164554:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763740.164561:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763740.164602:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763740.164617:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763740/real 1789763740] req@ffff9811e1a23800 x1876700049031808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763740.164656:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049031808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763740.164672:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763740.164678:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763740.164686:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763740.164699:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763740.164741:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763740.164758:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763740/real 1789763740] req@ffff9811e1a23480 x1876700049031936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763740.164774:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049031936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763740.164784:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763740.164788:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763740.164792:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049031936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763740.419572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763741.059743:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763741.443600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763742.083719:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763742.467589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763743.107658:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763743.491692:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763744.131654:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763744.515564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763745.155689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763745.283689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763745.283702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763745.283707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763745.283719:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3194 00000100:00080000:0.0:1789763745.283732:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763745.283787:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763745.283835:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049032064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763745.283878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763745.283885:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763745.283888:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763745.283905:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3194 00000100:00080000:0.0:1789763745.283912:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763745.283920:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763745.283933:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763745.283940:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049032064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763745.283955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049032192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763745.283972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763745.283976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763745.283978:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763745.283983:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3194 00000100:00080000:0.0:1789763745.283987:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763745.283992:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763745.283997:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763745.284010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049032320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763745.284020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763745.284025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789763745.284029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763745.284055:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763745.284064:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3194 00000100:00080000:0.0:1789763745.284072:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763745.284079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763745.284087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763745.284104:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049032448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763745.284116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763745.284121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763745.284124:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763745.284134:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3194 00000100:00100000:3.0:1789763745.284134:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763745/real 1789763745] req@ffff9811c2e29180 x1876700049032064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763745.284140:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763745.284149:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763745.284159:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763745.284200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049032576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763745.284236:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049032064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763745.284277:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763745.284283:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763745.284289:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049032064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763745.284364:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049032192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763745.284417:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763745.284437:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763745/real 1789763745] req@ffff9811c2e28700 x1876700049032192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763745.284508:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049032192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763745.284522:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763745.284543:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763745.284547:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049032192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763745.284556:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763745.284585:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763745.284599:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763745/real 1789763745] req@ffff9811c2e2ad80 x1876700049032320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763745.284613:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049032320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763745.284622:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763745.284626:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763745.284630:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763745.284636:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763745.284654:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763745.284664:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763745/real 1789763745] req@ffff9811c2e28e00 x1876700049032448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763745.284714:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049032448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763745.284742:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763745.284745:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763745.284749:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763745.284755:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763745.284778:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763745.284788:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763745/real 1789763745] req@ffff9811c2e29880 x1876700049032576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763745.284800:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049032576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763745.284808:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763745.284811:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763745.284814:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763745.539603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763746.179653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763746.563520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763747.203760:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763747.587601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763748.227707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763748.611582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763749.251731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763749.635591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763750.275709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763750.403655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763750.403672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763750.403681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763750.403702:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3199 00000100:00080000:0.0:1789763750.403724:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763750.403741:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763750.403808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049032704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763750.403874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763750.403880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763750.403883:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763750.403889:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3199 00000100:00080000:0.0:1789763750.403895:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763750.403902:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763750.403918:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763750.403939:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049032832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763750.403952:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049032704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763750.403961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763750.403969:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763750.403972:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763750.403978:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3199 00000100:00080000:0.0:1789763750.403983:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763750.403988:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763750.403995:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763750.404009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049032960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763750.404023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763750.404028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763750.404031:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763750.404035:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3199 00000100:00080000:0.0:1789763750.404050:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763750.404056:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763750.404064:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763750.404083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763750.404102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049033088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763750.404126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763750.404133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763750.404138:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763750.404147:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3199 00000100:00080000:0.0:1789763750.404157:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763750.404160:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763750/real 1789763750] req@ffff9811c2e29c00 x1876700049032704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763750.404199:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763750.404213:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763750.404251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049032704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763750.404268:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049033216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763750.404269:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763750.404282:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763750.404297:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049032704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763750.404343:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049032832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763750.404379:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763750.404397:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763750/real 1789763750] req@ffff9811c2e2bb80 x1876700049032832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763750.404413:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049032832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763750.404424:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763750.404428:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763750.404432:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049032832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763750.404439:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763750.404508:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763750.404533:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763750/real 1789763750] req@ffff9811c2e2a680 x1876700049032960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763750.404552:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049032960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763750.404582:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763750.404591:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763750.404597:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049032960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763750.404608:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763750.404664:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763750.404695:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763750/real 1789763750] req@ffff9811c2e2b100 x1876700049033088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763750.404722:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049033088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763750.404740:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763750.404747:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763750.404754:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763750.404765:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763750.404795:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763750.404810:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763750/real 1789763750] req@ffff9811c2e2aa00 x1876700049033216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763750.404825:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049033216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763750.404841:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763750.404847:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763750.404873:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763750.659564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763751.299733:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763751.683571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763752.323685:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763752.707646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763753.347705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763753.731591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763754.371662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763754.755614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763755.395697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763755.523629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763755.523645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763755.523653:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763755.523671:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3205 00000100:00080000:0.0:1789763755.523689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763755.523705:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763755.523776:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049033344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763755.523821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763755.523859:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763755.523874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763755.523885:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3205 00000100:00080000:0.0:1789763755.523895:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763755.523901:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049033344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763755.523911:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763755.523930:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763755.523962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049033472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763755.523989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763755.523998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1789763755.524001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763755.524005:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763755.524019:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3205 00000100:00080000:0.0:1789763755.524029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763755.524038:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763755.524048:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763755.524067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763755/real 1789763755] req@ffff9811c2e28380 x1876700049033344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763755.524079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049033600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763755.524107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763755.524115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763755.524121:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763755.524155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049033344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763755.524156:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3205 00000100:00080000:0.0:1789763755.524172:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763755.524183:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763755.524186:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:3.0:1789763755.524194:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763755.524202:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049033344:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763755.524219:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763755.524243:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049033472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763755.524254:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049033728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763755.524282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789763755.524290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763755.524297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763755.524306:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763755.524317:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3205 00000100:00100000:3.0:1789763755.524320:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763755/real 1789763755] req@ffff9811c2e28a80 x1876700049033472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763755.524328:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763755.524349:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763755.524353:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049033472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763755.524366:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1789763755.524372:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763755.524380:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763755.524400:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049033856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763755.524405:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049033472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763755.524429:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763755.524508:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763755.524528:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763755/real 1789763755] req@ffff9811c2e29f80 x1876700049033600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763755.524554:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049033600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763755.524570:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763755.524576:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763755.524583:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763755.524615:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763755.524654:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763755.524670:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763755/real 1789763755] req@ffff9811c2e2b800 x1876700049033728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763755.524686:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049033728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763755.524696:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763755.524700:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763755.524704:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763755.524711:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763755.524730:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763755.524757:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763755/real 1789763755] req@ffff9811c2e2b480 x1876700049033856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763755.524771:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049033856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763755.524782:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763755.524785:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763755.524789:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049033856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763755.779601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763756.419624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763756.803639:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763757.443751:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763757.827643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763758.467651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763758.851598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763759.491750:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763759.875533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763760.515718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763760.643747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763760.643765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763760.643773:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763760.643795:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3210 00000100:00080000:2.0:1789763760.643818:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763760.643833:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763760.643901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049033984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763760.643944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763760.643949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763760.643952:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763760.643957:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3210 00000100:00080000:2.0:1789763760.643963:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763760.643968:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763760.643982:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763760.643998:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049034112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763760.644016:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763760.644022:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763760.644024:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763760.644065:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3210 00000100:00100000:3.0:1789763760.644066:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049033984:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763760.644080:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763760.644115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763760.644136:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763760.644159:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049034240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763760.644177:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763760.644183:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763760.644186:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763760.644192:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3210 00000100:00080000:2.0:1789763760.644210:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763760.644219:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763760.644225:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763760.644256:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049034368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763760.644258:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763760.644284:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763760.644290:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763760.644293:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763760.644298:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3210 00000100:00080000:2.0:1789763760.644304:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763760.644309:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763760.644317:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763760.644342:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049034496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763760.644375:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763760/real 1789763760] req@ffff9811d863b800 x1876700049033984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763760.644408:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049033984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763760.644425:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763760.644431:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763760.644436:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049033984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763760.644446:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049034112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763760.644475:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763760.644559:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763760/real 1789763760] req@ffff9811d863a300 x1876700049034112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763760.644573:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049034112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763760.644582:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763760.644586:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763760.644589:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049034112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763760.644619:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763760.644643:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763760.644679:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763760/real 1789763760] req@ffff9811d863b480 x1876700049034240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763760.644704:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049034240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763760.644720:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763760.644727:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763760.644734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763760.644748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763760.644789:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763760.644807:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763760/real 1789763760] req@ffff9811d863aa00 x1876700049034368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763760.644823:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049034368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763760.644844:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763760.644848:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763760.644853:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763760.644861:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763760.644890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763760.644903:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763760/real 1789763760] req@ffff9811d8638000 x1876700049034496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763760.644917:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049034496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763760.644926:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763760.644930:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763760.644933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763760.899549:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763761.539604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763761.923521:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763762.563596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763762.947573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763763.587677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763763.971534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763764.611649:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763764.995582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763765.635694:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763765.763618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763765.763637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763765.763646:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763765.763666:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3215 00000100:00080000:1.0:1789763765.763690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763765.763706:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763765.763767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049034624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763765.763813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763765.763821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763765.763824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763765.763830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3215 00000100:00080000:1.0:1789763765.763858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763765.763868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763765.763885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763765.763914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049034752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763765.763915:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049034624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763765.763950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763765.763958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763765.763962:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763765.763969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3215 00000100:00080000:1.0:1789763765.763975:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763765.763982:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763765.763990:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763765.764008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049034880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763765.764022:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763765.764028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763765.764030:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763765.764035:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3215 00000100:00080000:1.0:1789763765.764041:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763765.764047:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763765.764080:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763765.764097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049035008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763765.766249:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763765.766255:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763765.766259:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763765.766272:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3215 00000100:00080000:1.0:1789763765.766286:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763765.766296:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763765.766312:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763765.766352:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049035136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763765.767957:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763765.768158:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763765/real 1789763765] req@ffff9811c92c6d80 x1876700049034624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763765.768203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049034624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763765.768236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763765.768243:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763765.768249:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049034624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763765.768291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049034752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763765.768349:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763765.768368:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763765/real 1789763765] req@ffff9811c92c7480 x1876700049034752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763765.768385:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049034752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763765.768396:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763765.768401:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763765.768406:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049034752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763765.768413:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763765.768435:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763765.768447:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763765/real 1789763765] req@ffff9811c92c4000 x1876700049034880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763765.768493:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049034880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763765.768506:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763765.768516:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763765.768521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049034880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763765.768544:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763765.768578:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763765.768592:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763765/real 1789763765] req@ffff9811c92c5500 x1876700049035008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763765.768608:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049035008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763765.768619:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763765.768623:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763765.768627:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763765.768634:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763765.768653:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763765.768664:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763765/real 1789763765] req@ffff9811c92c4e00 x1876700049035136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763765.768677:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049035136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763765.768687:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763765.768690:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763765.768694:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763766.019655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763766.659743:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763767.043542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763767.683671:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763768.067643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763768.707633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763769.091558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763769.731618:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763770.118556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763770.755712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763770.883604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763770.883613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763770.883617:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763770.883627:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3220 00000100:00080000:0.0:1789763770.883649:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763770.883656:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763770.883697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700049035264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763770.883732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763770.883735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763770.883736:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763770.883739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3220 00000100:00080000:0.0:1789763770.883756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763770.883761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763770.883770:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049035264:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763770.883771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763770.883787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700049035392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763770.883797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763770.883800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763770.883801:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763770.883804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3220 00000100:00080000:0.0:1789763770.883807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763770.883810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763770.883813:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763770.883822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700049035520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763770.883828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763770.883830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763770.883832:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763770.883834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3220 00000100:00080000:0.0:1789763770.883837:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763770.883840:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763770.883843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763770.883851:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763770.883853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049035648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763770.883871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763770.883874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763770.883875:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763770.883878:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3220 00000100:00080000:0.0:1789763770.883881:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763770.883884:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763770.883888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763770.883899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049035776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763770.883903:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763770/real 1789763770] req@ffff9811c2e29500 x1876700049035264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763770.883924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700049035264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763770.883934:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763770.883938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763770.883941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049035264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763770.883963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049035392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763770.883983:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763770.883992:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763770/real 1789763770] req@ffff9811c2e28000 x1876700049035392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763770.884000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700049035392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763770.884006:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763770.884009:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763770.884011:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049035392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763770.884015:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763770.884043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763770.884077:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763770/real 1789763770] req@ffff9811c2e2a300 x1876700049035520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763770.884092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700049035520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763770.884098:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763770.884100:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763770.884102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763770.884106:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763770.884120:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763770.884127:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763770/real 1789763770] req@ffff9811c2e2b480 x1876700049035648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763770.884134:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049035648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763770.884139:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763770.884140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763770.884142:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763770.884145:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763770.884155:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763770.884160:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763770/real 1789763770] req@ffff9811c2e2b800 x1876700049035776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763770.884167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049035776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763770.884180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763770.884183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763770.884185:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049035776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763771.139538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763771.779610:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763772.163573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763772.803626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763773.187542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763773.827643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763774.211611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763774.851616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763775.235599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763775.875677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763776.003575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763776.003586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763776.003591:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763776.003604:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3225 00000100:00080000:0.0:1789763776.003618:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763776.003652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763776.003702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049035904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763776.003732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763776.003736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763776.003738:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763776.003744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3225 00000100:00080000:0.0:1789763776.003750:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763776.003756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763776.003787:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049035904:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763776.003790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763776.003815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049036032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763776.003829:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763776.003833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763776.003835:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763776.003838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3225 00000100:00080000:0.0:1789763776.003843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763776.003847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763776.003852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763776.003863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049036160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763776.003872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789763776.003875:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763776.003877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763776.003880:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763776.003885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3225 00000100:00080000:0.0:1789763776.003888:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763776.003892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763776.003896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763776.003906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049036288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763776.003913:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763776.003917:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763776.003925:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763776.003928:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763776/real 1789763776] req@ffff9811c2e29f80 x1876700049035904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763776.003933:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3225 00000100:00080000:0.0:1789763776.003941:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763776.003948:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763776.003954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763776.003958:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049035904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763776.003968:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049036416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763776.003974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763776.003998:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763776.004004:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049035904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763776.004036:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049036032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763776.004060:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763776.004071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763776/real 1789763776] req@ffff9811c2e28a80 x1876700049036032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763776.004092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049036032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763776.004100:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763776.004103:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763776.004105:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049036032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763776.004110:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763776.004127:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763776.004135:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763776/real 1789763776] req@ffff9811c2e28380 x1876700049036160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763776.004145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049036160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763776.004152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763776.004154:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763776.004156:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763776.004160:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763776.004175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763776.004181:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763776/real 1789763776] req@ffff9811c2e2aa00 x1876700049036288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763776.004191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049036288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763776.004208:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763776.004211:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763776.004214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763776.004218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763776.004234:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763776.004242:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763776/real 1789763776] req@ffff9811c2e2b100 x1876700049036416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763776.004253:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049036416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763776.004263:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763776.004268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763776.004272:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763776.259534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763776.899661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763777.285172:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763777.923589:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763778.307604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763778.947624:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763779.331551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763779.971627:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763780.355599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763780.995579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763781.123538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763781.123548:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763781.123551:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763781.123561:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3230 00000100:00080000:0.0:1789763781.123571:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763781.123578:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763781.123623:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049036544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763781.123656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763781.123659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763781.123661:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763781.123664:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3230 00000100:00080000:0.0:1789763781.123673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763781.123676:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763781.123685:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763781.123696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049036672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763781.123708:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049036544:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763781.123729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763781.123732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763781.123734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763781.123737:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3230 00000100:00080000:0.0:1789763781.123741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763781.123744:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763781.123748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763781.123757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049036800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763781.123763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763781.123766:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763781.123767:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763781.123769:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3230 00000100:00080000:0.0:1789763781.123772:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763781.123773:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763781.123777:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763781.123781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763781.123788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049036928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763781.123795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763781.123798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763781.123799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763781.123801:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3230 00000100:00080000:0.0:1789763781.123811:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763781.123815:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763781.123820:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763781/real 1789763781] req@ffff9811c2e2a680 x1876700049036544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763781.123821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763781.123834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049037056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763781.123869:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049036544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763781.123879:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763781.123882:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763781.123886:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049036544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763781.123907:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049036672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763781.123924:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763781.123932:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763781/real 1789763781] req@ffff9811c2e2bb80 x1876700049036672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763781.123940:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049036672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763781.123959:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763781.123962:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763781.123964:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049036672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763781.123968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763781.123980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763781.123988:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763781/real 1789763781] req@ffff9811c2e29c00 x1876700049036800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763781.123995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049036800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763781.124012:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763781.124014:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763781.124016:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763781.124020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763781.124034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763781.124041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763781/real 1789763781] req@ffff9811c2e29880 x1876700049036928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763781.124048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049036928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763781.124054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763781.124055:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763781.124057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049036928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763781.124060:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763781.124070:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763781.124075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763781/real 1789763781] req@ffff9811c2e28e00 x1876700049037056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763781.124082:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049037056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763781.124087:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763781.124089:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763781.124096:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763781.379578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763782.019670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763782.403532:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763783.043596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763783.427548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763784.067579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763784.451589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763785.091584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763785.475536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763786.115630:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763786.243625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763786.243635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763786.243639:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763786.243649:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3235 00000100:00080000:3.0:1789763786.243677:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763786.243686:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763786.243749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049037184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763786.243792:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763786.243795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763786.243797:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763786.243800:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3235 00000100:00080000:3.0:1789763786.243804:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763786.243808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763786.243825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763786.243827:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049037184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763786.243838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049037312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763786.243889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763786.243891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763786.243893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763786.243896:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3235 00000100:00080000:3.0:1789763786.243900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763786.243917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763786.243922:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763786.243929:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763786.243935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049037440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763786.243960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763786.243963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763786.243965:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763786.243968:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3235 00000100:00080000:3.0:1789763786.243971:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763786.243973:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763786.243978:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763786.243987:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049037568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763786.243995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763786/real 1789763786] req@ffff9811e1a21500 x1876700049037184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763786.243996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763786.244004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763786.244007:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763786.244025:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3235 00000100:00100000:0.0:1789763786.244033:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049037184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763786.244034:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763786.244042:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763786.244049:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763786.244049:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763786.244057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763786.244061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049037184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763786.244066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049037696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763786.244195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049037312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763786.244264:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763786.244290:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763786/real 1789763786] req@ffff9811e1a20700 x1876700049037312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763786.244305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049037312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763786.244313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763786.244316:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763786.244333:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049037312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763786.244341:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763786.244365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763786.244378:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763786/real 1789763786] req@ffff9811e1a20a80 x1876700049037440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763786.244401:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049037440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763786.244409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763786.244411:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763786.244413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763786.244418:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763786.244435:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763786.244443:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763786/real 1789763786] req@ffff9811e1a21f80 x1876700049037568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763786.244451:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049037568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763786.244456:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763786.244458:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763786.244499:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763786.244504:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763786.244523:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763786.244537:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763786/real 1789763786] req@ffff9811e1a20000 x1876700049037696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763786.244545:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049037696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763786.244560:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763786.244562:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763786.244564:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049037696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763786.499591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763787.139588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763787.523567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763788.163641:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763788.547530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763789.187600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763789.571537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763790.211614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763790.595543:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763791.235584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763791.363561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763791.363570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763791.363574:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763791.363583:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3240 00000100:00080000:0.0:1789763791.363594:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763791.363600:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763791.363637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049037824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763791.363662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763791.363665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763791.363667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763791.363670:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3240 00000100:00080000:0.0:1789763791.363672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763791.363685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763791.363695:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763791.363696:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049037824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763791.363707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049037952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763791.363736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763791.363739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763791.363741:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763791.363744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3240 00000100:00080000:0.0:1789763791.363748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763791.363751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763791.363755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763791.363765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049038080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763791.363772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763791.363775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763791.363776:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763791.363779:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3240 00000100:00080000:0.0:1789763791.363781:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763791.363784:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763791.363788:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763791.363796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049038208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763791.363804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763791.363805:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049037952:192.168.203.113@tcp:38:kworker.0 00000100:00080000:0.0:1789763791.363808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763791.363812:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763791.363816:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3240 00000100:00080000:0.0:1789763791.363823:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763791.363826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763791.363831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763791.363846:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763791.363847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049038336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763791.363897:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763791.363913:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763791.619536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763792.259595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763792.643526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763793.283579:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763793.667577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763794.307611:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763794.435964:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763794.436016:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763794.436053:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763794.436096:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763794.436110:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763794.436122:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763794.436138:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763791/real 1789763794] req@ffff9811c2e28700 x1876700049037952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789763794.436150:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763794.436183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049037952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763794.436200:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763794.436207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763794.436212:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049037952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763794.436222:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763791/real 1789763794] req@ffff9811c2e29180 x1876700049038080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763794.436237:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049038080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763794.436246:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763794.436250:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763794.436254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763794.436262:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763791/real 1789763794] req@ffff9811c2e2ad80 x1876700049037824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763794.436293:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049037824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763794.436302:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763794.436306:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763794.436310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049037824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763794.436339:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763791/real 1789763794] req@ffff9811c2e28e00 x1876700049038208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763794.436354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049038208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763794.436364:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763794.436368:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763794.436372:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763794.436379:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763791/real 1789763794] req@ffff9811c2e29880 x1876700049038336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763794.436393:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049038336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763794.436400:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763794.436403:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763794.436408:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763794.691551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763795.331597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763795.715540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763796.355613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763796.483594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763796.483607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763796.483611:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763796.483620:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3246 00000100:00080000:3.0:1789763796.483630:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763796.483637:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763796.483670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049038464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763796.483696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763796.483699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763796.483700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763796.483703:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3246 00000100:00080000:3.0:1789763796.483706:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763796.483708:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763796.483717:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763796.483725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049038592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763796.483734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763796.483737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763796.483738:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763796.483741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3246 00000100:00080000:3.0:1789763796.483744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763796.483762:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049038464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763796.483766:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763796.483772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763796.483781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049038720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763796.483802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763796.483806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763796.483807:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763796.483814:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3246 00000100:00080000:3.0:1789763796.483817:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763796.483820:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763796.483823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763796.483846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049038848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763796.483854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763796.483858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763796.483860:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763796.483868:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3246 00000100:00080000:3.0:1789763796.483874:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763796.483879:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763796.483887:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763796.483897:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763796.483915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049038976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763796.483940:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763796/real 1789763796] req@ffff9811e1a20e00 x1876700049038464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763796.483970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049038464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763796.483982:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763796.483987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763796.483991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049038464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763796.483999:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049038592:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763796.484025:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763796.484039:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763796/real 1789763796] req@ffff9811e1a20380 x1876700049038592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763796.484051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049038592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763796.484059:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763796.484063:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763796.484066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049038592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763796.484095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038720:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763796.484117:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763796.484129:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763796/real 1789763796] req@ffff9811e1a20000 x1876700049038720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763796.484161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049038720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763796.484170:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763796.484174:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763796.484178:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763796.484185:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763796.484226:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763796.484239:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763796/real 1789763796] req@ffff9811e1a21f80 x1876700049038848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763796.484252:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049038848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763796.484261:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763796.484265:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763796.484268:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763796.484275:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763796.484297:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763796.484308:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763796/real 1789763796] req@ffff9811e1a20a80 x1876700049038976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763796.484319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049038976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763796.484340:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763796.484344:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763796.484348:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049038976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763796.739541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763797.379591:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763797.763516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763798.403650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763798.787500:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763799.427584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763799.811507:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763800.451599:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763800.835503:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763801.475559:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763801.603554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763801.603565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763801.603571:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763801.603584:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3251 00000100:00080000:3.0:1789763801.603597:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763801.603606:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763801.603648:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049039104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763801.603681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763801.603686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763801.603689:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763801.603695:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3251 00000100:00080000:3.0:1789763801.603700:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763801.603705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763801.603717:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763801.603734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049039232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763801.603745:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049039104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763801.603753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763801.603759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763801.603761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763801.603767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3251 00000100:00080000:3.0:1789763801.603786:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763801.603835:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763801.603880:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763801/real 1789763801] req@ffff9811e1a20700 x1876700049039104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763801.603891:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763801.603901:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763801.603902:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049039104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.603930:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.603934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763801.603939:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049039104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763801.603962:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049039232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763801.603988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763801.604003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763801/real 1789763801] req@ffff9811e1a21500 x1876700049039232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.604018:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049039232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.604028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.604032:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763801.604037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049039360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763801.604040:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049039232:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763801.604107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763801.604112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763801.604114:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763801.604118:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3251 00000100:00080000:3.0:1789763801.604123:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763801.604128:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763801.604135:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763801.604143:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763801.604151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049039488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763801.604164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763801.604168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763801.604171:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763801.604175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763801.604180:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3251 00000100:00080000:3.0:1789763801.604186:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763801.604192:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763801.604197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763801/real 1789763801] req@ffff9811e1a23480 x1876700049039360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763801.604199:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763801.604220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049039616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763801.604221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049039360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.604249:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.604254:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763801.604262:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763801.604289:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763801.604314:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763801.604326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763801/real 1789763801] req@ffff9811e1a23800 x1876700049039488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.604339:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049039488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.604348:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.604352:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763801.604355:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763801.604363:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763801.604383:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763801.604395:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763801/real 1789763801] req@ffff9811e1a23100 x1876700049039616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.604408:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049039616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.604416:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.604419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763801.604423:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049039616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763801.859520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763802.499580:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763802.883506:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763803.523553:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763803.907544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763804.547614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763804.931527:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763805.571641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763805.955563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763806.595608:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763806.723604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763806.723615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763806.723619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763806.723629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3256 00000100:00080000:3.0:1789763806.723639:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763806.723646:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763806.723683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049039744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763806.723709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763806.723712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763806.723713:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763806.723716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3256 00000100:00080000:3.0:1789763806.723719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763806.723722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763806.723731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763806.723739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049039872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763806.723746:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049039744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763806.723750:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763806.723785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763806.723787:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763806.723790:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3256 00000100:00080000:3.0:1789763806.723794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763806.723797:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763806.723802:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763806.723813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049040000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763806.723821:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763806.723823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763806.723825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763806.723827:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3256 00000100:00080000:3.0:1789763806.723830:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763806.723832:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763806.723836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763806.723843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049040128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763806.723848:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763806.723855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763806.723858:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763806.723860:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763806.723864:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3256 00000100:00080000:3.0:1789763806.723867:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763806.723869:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763806.723879:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763806.723889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049040256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763806.723895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763806/real 1789763806] req@ffff9811e1a22680 x1876700049039744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763806.723913:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049039744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763806.723923:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763806.723926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763806.723929:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049039744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763806.723949:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049039872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763806.723964:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763806.723972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763806/real 1789763806] req@ffff9811e1a21c00 x1876700049039872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763806.723981:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049039872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763806.723986:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763806.723987:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763806.723989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049039872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763806.723993:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763806.724003:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763806.724009:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763806/real 1789763806] req@ffff9811e1a21880 x1876700049040000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763806.724027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049040000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763806.724033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763806.724035:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763806.724037:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763806.724041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763806.724054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763806.724060:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763806/real 1789763806] req@ffff9811e1a22300 x1876700049040128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763806.724067:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049040128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763806.724072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763806.724074:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763806.724075:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763806.724078:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763806.724088:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763806.724093:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763806/real 1789763806] req@ffff9811e1a21180 x1876700049040256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763806.724099:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049040256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763806.724112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763806.724115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763806.724117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763806.979578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763807.619585:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763808.003520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763808.643577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763809.027506:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763809.667584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763810.051533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763810.691639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763811.075604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763811.715672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763811.843574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763811.843586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763811.843592:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763811.843616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3261 00000100:00080000:3.0:1789763811.843630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763811.843640:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763811.843702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049040384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763811.843767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763811.843772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763811.843774:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763811.843779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3261 00000100:00080000:3.0:1789763811.843784:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763811.843789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763811.843801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763811.843820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049040512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763811.843859:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049040384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763811.843869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763811.843876:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763811.843878:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763811.843885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3261 00000100:00080000:3.0:1789763811.843892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763811.843897:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763811.843904:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763811.843941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049040640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763811.843955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763811.843960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763811.843962:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763811.843967:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3261 00000100:00080000:3.0:1789763811.843973:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763811.843980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763811.843986:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763811.843996:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763811.844014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049040768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763811.844027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763811.844031:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763811.844035:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763811.844042:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3261 00000100:00100000:1.0:1789763811.844048:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763811/real 1789763811] req@ffff9811e1a22d80 x1876700049040384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763811.844069:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763811.844086:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763811.844088:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049040384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763811.844106:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1789763811.844112:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763811.844121:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763811.844127:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049040384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763811.844142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049040896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763811.844170:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049040512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763811.844205:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763811.844221:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763811/real 1789763811] req@ffff9811e1a22a00 x1876700049040512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763811.844261:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049040512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763811.844275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763811.844296:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763811.844321:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049040512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763811.844334:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763811.844394:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763811.844432:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763811/real 1789763811] req@ffff9811e1a21180 x1876700049040640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763811.844447:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049040640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763811.844456:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763811.844460:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763811.844464:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763811.844470:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763811.844534:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763811.844548:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763811/real 1789763811] req@ffff9811e1a22300 x1876700049040768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763811.844562:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049040768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763811.844571:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763811.844585:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763811.844589:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763811.844595:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763811.844649:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763811.844661:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763811/real 1789763811] req@ffff9811e1a21880 x1876700049040896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763811.844676:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049040896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763811.844685:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763811.844689:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763811.844693:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049040896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763812.099537:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763812.739593:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763813.123664:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763813.763688:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763814.147624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763814.787619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763815.171619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763815.811741:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763816.195531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763816.835675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763816.963648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763816.963666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763816.963675:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763816.963729:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3266 00000100:00080000:1.0:1789763816.963743:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763816.963751:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763816.963803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049041024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763816.963835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763816.963839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763816.963841:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763816.963846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3266 00000100:00080000:1.0:1789763816.963852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763816.963870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763816.963874:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049041024:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789763816.963884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763816.963903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049041152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763816.963922:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763816.963927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763816.963930:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763816.963934:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3266 00000100:00080000:1.0:1789763816.963943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763816.963950:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763816.963953:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763816.963958:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763816.963976:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049041280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763816.963987:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763816.963991:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763816.963994:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763816.964000:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763816/real 1789763816] req@ffff9811c92c5180 x1876700049041024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763816.964001:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3266 00000100:00080000:1.0:1789763816.964008:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763816.964016:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763816.964023:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763816.964053:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049041024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763816.964072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049041408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763816.964088:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763816.964099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763816.964106:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763816.964115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:1.0:1789763816.964116:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763816.964143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049041024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763816.964144:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3266 00000100:00080000:1.0:1789763816.964152:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763816.964159:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763816.964166:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763816.964180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049041152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763816.964182:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049041536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763816.964213:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763816.964225:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763816/real 1789763816] req@ffff9811c92c5880 x1876700049041152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763816.964246:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049041152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763816.964252:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763816.964254:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763816.964257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049041152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763816.964261:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763816.964281:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763816.964293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763816/real 1789763816] req@ffff9811c92c7b80 x1876700049041280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763816.964306:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049041280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763816.964315:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763816.964318:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763816.964322:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763816.964328:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763816.964350:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763816.964362:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763816/real 1789763816] req@ffff9811c92c4e00 x1876700049041408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763816.964371:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049041408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763816.964386:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763816.964390:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763816.964393:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763816.964411:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763816.964433:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763816.964445:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763816/real 1789763816] req@ffff9811c92c5500 x1876700049041536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763816.964457:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049041536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763816.964512:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763816.964518:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763816.964522:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763817.219721:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763817.859629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763818.243600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763818.883813:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763819.267577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763819.907680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763820.291539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763820.931666:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763821.315574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763821.955615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763822.083598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763822.083609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763822.083613:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763822.083623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3271 00000100:00080000:1.0:1789763822.083634:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763822.083657:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763822.083695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049041664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763822.083732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763822.083735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763822.083737:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763822.083739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3271 00000100:00080000:1.0:1789763822.083742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763822.083745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763822.083753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763822.083763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049041792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763822.083790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763822.083793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763822.083795:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763822.083798:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3271 00000100:00080000:1.0:1789763822.083803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763822.083806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763822.083810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763822.083820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049041920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763822.083829:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049041664:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763822.083833:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763822.083850:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763822.083852:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763822.083856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3271 00000100:00080000:1.0:1789763822.083858:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763822.083860:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763822.083863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763822.083873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049042048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763822.083879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763822.083881:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763822.083882:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763822.083885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3271 00000100:00080000:1.0:1789763822.083896:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763822.083900:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763822.083906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763822.083920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049042176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763822.083956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763822.084015:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763822/real 1789763822] req@ffff9811c92c4000 x1876700049041664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763822.084039:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049041664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763822.084070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763822.084076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763822.084080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049041664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763822.084092:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049041792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763822.084134:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763822.084170:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763822/real 1789763822] req@ffff9811c92c7480 x1876700049041792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763822.084185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049041792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763822.084195:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763822.084199:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763822.084203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049041792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763822.084266:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763822.084294:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763822.084324:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763822/real 1789763822] req@ffff9811c92c6d80 x1876700049041920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763822.084332:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049041920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763822.084338:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763822.084341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763822.084343:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049041920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763822.084354:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763822.084387:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763822.084399:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763822/real 1789763822] req@ffff9811c92c5c00 x1876700049042048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763822.084412:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049042048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763822.084420:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763822.084423:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763822.084426:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763822.084432:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763822.084449:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763822.084459:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763822/real 1789763822] req@ffff9811c92c6680 x1876700049042176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763822.084470:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049042176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763822.084478:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763822.084481:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763822.084484:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763822.339553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763822.979712:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763823.363558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763824.003598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763824.387535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763825.027698:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763825.411540:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763826.051604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763826.435524:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763827.075660:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763827.203637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763827.203648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763827.203652:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763827.203663:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3276 00000100:00080000:1.0:1789763827.203675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763827.203683:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763827.203724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049042304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763827.203747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763827.203750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763827.203752:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763827.203755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3276 00000100:00080000:1.0:1789763827.203758:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763827.203760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763827.203769:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763827.203778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049042432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763827.203791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763827.203801:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049042304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763827.203812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763827.203814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763827.203818:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3276 00000100:00080000:1.0:1789763827.203821:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763827.203825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763827.203838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763827.203853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049042560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763827.203861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763827.203864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763827.203865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763827.203868:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3276 00000100:00080000:1.0:1789763827.203871:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763827.203873:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763827.203879:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763827.203887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049042688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763827.203894:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763827.203897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763827.203899:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763827.203905:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763827.203910:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3276 00000100:00080000:1.0:1789763827.203915:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763827.203918:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763827.203928:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763827.203942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049042816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763827.203959:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763827/real 1789763827] req@ffff9811c92c4380 x1876700049042304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763827.203978:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049042304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763827.203987:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763827.203991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763827.203994:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049042304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763827.204016:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049042432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763827.204032:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763827.204040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763827/real 1789763827] req@ffff9811c92c4a80 x1876700049042432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763827.204048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049042432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763827.204054:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763827.204056:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763827.204058:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049042432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763827.204062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763827.204074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763827.204080:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763827/real 1789763827] req@ffff9811c92c5f80 x1876700049042560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763827.204098:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049042560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763827.204104:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763827.204106:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763827.204108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763827.204112:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763827.204126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763827.204132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763827/real 1789763827] req@ffff9811c92c7800 x1876700049042688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763827.204139:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049042688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763827.204144:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763827.204146:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763827.204148:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763827.204151:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763827.204161:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763827.204167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763827/real 1789763827] req@ffff9811c92c4700 x1876700049042816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763827.204173:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049042816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763827.204186:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763827.204188:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763827.204190:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049042816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763827.459554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763828.099692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763828.483533:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763829.123813:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763829.507568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763830.147622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763830.531619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763831.171676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763831.555586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763832.195696:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763832.323691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763832.323704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763832.323709:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763832.323738:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3281 00000100:00080000:3.0:1789763832.323748:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763832.323755:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763832.323819:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049042944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763832.323863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763832.323869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763832.323871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763832.323876:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3281 00000100:00080000:3.0:1789763832.323881:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763832.323886:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763832.323898:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763832.323914:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049043072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763832.323929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763832.323935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763832.323938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763832.323942:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3281 00000100:00080000:3.0:1789763832.323946:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763832.323951:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763832.323971:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049042944:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763832.323976:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763832.324000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049043200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763832.324016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763832.324020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763832.324023:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763832.324043:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3281 00000100:00080000:3.0:1789763832.324046:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763832.324049:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763832.324052:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763832.324060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049043328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763832.324067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763832.324069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763832.324070:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763832.324073:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3281 00000100:00080000:3.0:1789763832.324075:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763832.324077:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763832.324080:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763832.324090:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049043456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763832.324091:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763832.324139:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763832/real 1789763832] req@ffff9811e1a21c00 x1876700049042944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763832.324155:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049042944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763832.324165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763832.324168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763832.324171:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049042944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763832.324177:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049043072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763832.324223:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763832.324240:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763832/real 1789763832] req@ffff9811e1a22680 x1876700049043072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763832.324265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049043072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763832.324275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763832.324292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763832.324297:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049043072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763832.324348:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763832.324392:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763832.324424:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763832/real 1789763832] req@ffff9811e1a23100 x1876700049043200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763832.324448:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049043200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763832.324460:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763832.324464:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763832.324468:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763832.324485:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763832.324556:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763832.324582:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763832/real 1789763832] req@ffff9811e1a23800 x1876700049043328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763832.324593:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049043328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763832.324599:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763832.324601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763832.324604:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763832.324630:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763832.324671:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763832.324686:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763832/real 1789763832] req@ffff9811e1a23480 x1876700049043456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763832.324701:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049043456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763832.324711:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763832.324731:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763832.324734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763832.579666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763833.219702:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763833.603642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763834.243730:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763834.627599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763835.267590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763835.651529:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763836.291783:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763836.675657:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763837.315740:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763837.443692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763837.443702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763837.443706:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763837.443718:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3287 00000100:00080000:1.0:1789763837.443730:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763837.443740:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763837.443783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049043584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763837.443818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763837.443822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763837.443825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763837.443831:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3287 00000100:00080000:1.0:1789763837.443836:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763837.443841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763837.443853:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763837.443875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700049043712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763837.443876:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049043584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763837.443904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763837.443909:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763837.443912:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763837.443918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3287 00000100:00080000:1.0:1789763837.443937:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763837.443942:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763837.443950:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763837.443965:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049043840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763837.443978:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763837.443990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763837.443995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763837.443997:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763837.444004:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3287 00000100:00080000:1.0:1789763837.444011:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763837.444016:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763837.444021:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763837.444030:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049043968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763837.444039:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763837.444042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763837.444043:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763837.444046:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3287 00000100:00080000:1.0:1789763837.444049:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763837.444051:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763837.444055:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763837.444063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049044096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763837.444097:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763837/real 1789763837] req@ffff9811c92c6300 x1876700049043584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763837.444118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049043584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763837.444134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763837.444268:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763837.444273:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049043584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763837.444355:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049043712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763837.444388:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763837.444406:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763837/real 1789763837] req@ffff9811c92c7100 x1876700049043712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763837.444417:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700049043712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763837.444425:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763837.444428:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763837.444431:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049043712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763837.444436:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763837.444500:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763837.444519:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763837/real 1789763837] req@ffff9811c92c6a00 x1876700049043840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763837.444544:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049043840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763837.444550:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763837.444553:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763837.444555:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763837.444560:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763837.444580:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763837.444588:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763837/real 1789763837] req@ffff9811c92c4700 x1876700049043968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763837.444596:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049043968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763837.444601:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763837.444603:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763837.444605:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049043968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763837.444609:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763837.444620:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763837.444626:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763837/real 1789763837] req@ffff9811c92c7800 x1876700049044096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763837.444645:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049044096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763837.444651:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763837.444672:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763837.444676:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763837.699651:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763838.339689:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763838.723577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763839.363651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763839.747582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763840.387677:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763840.771621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763841.411762:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763841.795633:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763842.435722:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763842.563693:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763842.563708:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763842.563713:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763842.563726:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3292 00000100:00080000:0.0:1789763842.563742:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763842.563752:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763842.563804:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049044224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763842.563853:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763842.563858:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763842.563860:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763842.563865:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3292 00000100:00080000:0.0:1789763842.563892:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763842.563899:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763842.563913:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763842.563930:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049044352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763842.563954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049044224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763842.563966:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763842.563973:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763842.563977:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763842.563988:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3292 00000100:00080000:0.0:1789763842.563997:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763842.564006:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763842.564015:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763842.564042:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049044480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763842.564063:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763842.564070:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763842.564073:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763842.564081:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3292 00000800:00000100:3.0:1789763842.564095:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763842.564098:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763842.564110:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763842.564119:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763842.564139:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049044608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763842.564168:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763842.564173:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763842.564175:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763842.564181:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3292 00000100:00080000:0.0:1789763842.564188:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763842.564193:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763842.564201:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763842.564203:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763842/real 1789763842] req@ffff9811c2e29880 x1876700049044224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763842.564228:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049044736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763842.564248:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049044224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763842.564275:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763842.564285:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763842.564295:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049044224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763842.564344:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049044352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763842.564387:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763842.564407:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763842/real 1789763842] req@ffff9811c2e28e00 x1876700049044352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763842.564561:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049044352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763842.564576:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763842.564581:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763842.564586:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049044352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763842.564595:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763842.564629:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763842.564646:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763842/real 1789763842] req@ffff9811c2e2ad80 x1876700049044480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763842.564661:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049044480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763842.564670:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763842.564674:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763842.564678:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763842.564685:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763842.564705:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763842.564717:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763842/real 1789763842] req@ffff9811c2e29180 x1876700049044608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763842.564737:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049044608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763842.564776:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763842.564783:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763842.564790:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763842.564803:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763842.564861:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763842.564879:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763842/real 1789763842] req@ffff9811c2e28700 x1876700049044736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763842.564894:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049044736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763842.564904:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763842.564908:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763842.564912:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049044736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763842.819607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763843.459802:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763843.843621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763844.483750:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763844.867612:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763845.507656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763845.891561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763846.531667:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763846.915607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763847.555836:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763847.683680:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763847.683692:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763847.683696:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763847.683707:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3297 00000100:00080000:3.0:1789763847.683719:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763847.683728:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763847.683807:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049044864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763847.683841:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763847.683844:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763847.683846:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763847.683849:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3297 00000100:00080000:3.0:1789763847.683853:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763847.683856:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763847.683865:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763847.683876:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049044992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763847.683886:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763847.683890:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763847.683891:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763847.683894:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3297 00000100:00080000:3.0:1789763847.683897:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763847.683900:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763847.683904:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763847.683910:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049044864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763847.683915:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049045120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763847.683925:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763847.683927:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763847.683928:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763847.683938:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3297 00000100:00080000:3.0:1789763847.683942:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763847.683945:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763847.683950:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763847.683960:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049045248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763847.683968:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763847.683971:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763847.683972:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763847.683975:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3297 00000100:00080000:3.0:1789763847.683978:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763847.683980:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763847.683983:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763847.683988:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763847.683996:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049045376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763847.684053:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763847/real 1789763847] req@ffff9811e1a21500 x1876700049044864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763847.684074:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049044864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763847.684085:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763847.684098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763847.684102:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049044864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763847.684108:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049044992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763847.684125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763847.684133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763847/real 1789763847] req@ffff9811e1a20700 x1876700049044992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763847.684153:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049044992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763847.684160:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763847.684163:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763847.684165:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049044992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763847.684188:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763847.684203:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763847.684211:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763847/real 1789763847] req@ffff9811e1a20a80 x1876700049045120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763847.684220:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049045120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763847.684226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763847.684228:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763847.684230:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763847.684234:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763847.684257:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763847.684263:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763847/real 1789763847] req@ffff9811e1a21f80 x1876700049045248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763847.684272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049045248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763847.684289:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763847.684292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763847.684294:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763847.684299:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763847.684314:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763847.684321:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763847/real 1789763847] req@ffff9811e1a20000 x1876700049045376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763847.684330:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049045376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763847.684336:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763847.684338:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763847.684340:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763847.939671:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763848.579751:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763848.963551:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763849.603709:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763849.987613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763850.627727:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763851.011547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763851.651734:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763852.035597:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763852.675774:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763852.803604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763852.803619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763852.803624:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763852.803639:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3302 00000100:00080000:3.0:1789763852.803685:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763852.803777:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763852.803837:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049045504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763852.803895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763852.803903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763852.803907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763852.803916:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3302 00000100:00080000:3.0:1789763852.803924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763852.803933:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763852.803953:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763852.803983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049045632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763852.804007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763852.804015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763852.804027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049045504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763852.804027:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763852.804064:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3302 00000100:00080000:3.0:1789763852.804077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763852.804090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763852.804103:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763852.804152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049045760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763852.804168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763852.804169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763852.804180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763852.804223:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763852.804233:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3302 00000100:00080000:3.0:1789763852.804242:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763852.804250:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763852.804255:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763852/real 1789763852] req@ffff9811e1a20380 x1876700049045504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763852.804281:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763852.804319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049045504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763852.804338:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049045888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763852.804361:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763852.804375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763852.804382:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763852.804392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789763852.804406:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049045504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763852.804406:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763852.804546:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3302 00000100:00100000:1.0:1789763852.804556:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049045632:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763852.804563:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763852.804578:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763852.804591:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:1.0:1789763852.804612:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763852.804622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049046016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763852.804638:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763852/real 1789763852] req@ffff9811e1a20e00 x1876700049045632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763852.804697:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049045632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763852.804710:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763852.804727:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763852.804732:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049045632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763852.804741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763852.804769:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763852.804783:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763852/real 1789763852] req@ffff9811e1a20000 x1876700049045760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763852.804797:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049045760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763852.804806:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763852.804810:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763852.804814:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763852.804821:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763852.804841:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763852.804851:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763852/real 1789763852] req@ffff9811e1a21f80 x1876700049045888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763852.804864:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049045888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763852.804890:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763852.804895:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763852.804899:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049045888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763852.804922:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763852.804947:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763852.804959:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763852/real 1789763852] req@ffff9811e1a20a80 x1876700049046016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763852.804973:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049046016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763852.804983:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763852.804986:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763852.804990:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763853.059587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763853.699679:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763854.083584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763854.723687:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763855.107602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763855.747615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763856.131643:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763856.771711:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763857.155574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763857.795802:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763857.923619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763857.923636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763857.923642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763857.923657:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3307 00000100:00080000:1.0:1789763857.923673:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763857.923686:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763857.923755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049046144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763857.923832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763857.923841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763857.923845:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763857.923854:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3307 00000100:00080000:1.0:1789763857.923864:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763857.923875:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763857.923895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763857.923905:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049046144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763857.923929:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049046272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763857.923950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763857.923958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763857.923963:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763857.923971:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3307 00000100:00080000:1.0:1789763857.923997:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763857.924008:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763857.924020:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763857.924049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049046400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763857.924077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049046272:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763857.924108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763857.924117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763857.924122:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763857.924134:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3307 00000100:00100000:0.0:1789763857.924147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046400:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789763857.924158:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763857.924182:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763857.924194:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763857.924217:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049046528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763857.924243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763857.924249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763857.924252:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763857.924259:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3307 00000100:00080000:1.0:1789763857.924266:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763857.924271:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763857.924279:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763857.924280:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763857.924301:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049046656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763857.924357:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763858.179593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763858.819584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763859.203550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763859.843689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763860.227553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763860.867626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763860.995788:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763860.995857:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763860.995879:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763860.995929:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763860.995939:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763860.995949:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763860.995957:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763860.996076:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763857/real 1789763860] req@ffff9811c92c5c00 x1876700049046656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763860.996202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049046656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763860.996226:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763860.996247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763860.996257:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763860.996275:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763857/real 1789763860] req@ffff9811c92c6680 x1876700049046528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763860.996302:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049046528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763860.996318:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763860.996325:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763860.996331:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763860.996345:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763857/real 1789763860] req@ffff9811c92c4380 x1876700049046400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763860.996400:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049046400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763860.996416:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763860.996422:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763860.996428:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049046400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763860.996442:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763857/real 1789763860] req@ffff9811c92c4a80 x1876700049046272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763860.996572:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049046272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763860.996592:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763860.996598:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763860.996605:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049046272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763860.996619:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763857/real 1789763860] req@ffff9811c92c5f80 x1876700049046144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763860.996645:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049046144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763860.996661:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763860.996667:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763860.996673:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049046144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763861.251589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763861.891695:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763862.275574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763862.915633:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763863.043617:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763863.043628:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763863.043632:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763863.043644:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3312 00000100:00080000:3.0:1789763863.043657:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763863.043665:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763863.043710:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049046784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763863.043755:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763863.043761:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763863.043764:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763863.043770:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3312 00000100:00080000:3.0:1789763863.043776:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763863.043781:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763863.043796:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763863.043821:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049046784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763863.043844:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049046912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763863.043868:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763863.043874:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763863.043877:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763863.043883:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3312 00000100:00080000:3.0:1789763863.043891:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763863.043897:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763863.043906:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763863.043924:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049047040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763863.043940:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763863.043944:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763863.043952:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763863.043958:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763863.043967:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3312 00000100:00080000:3.0:1789763863.043976:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763863.043984:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763863.043993:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763863.044019:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049047168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763863.044026:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763863/real 1789763863] req@ffff9811e1a20700 x1876700049046784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763863.044052:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763863.044060:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763863.044063:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763863.044076:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049046784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763863.044084:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3312 00000100:00080000:0.0:1789763863.044107:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763863.044108:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763863.044120:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763863.044130:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763863.044132:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049046784:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763863.044143:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763863.044175:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049047296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763863.044201:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049046912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763863.044267:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763863.044292:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763863/real 1789763863] req@ffff9811e1a21500 x1876700049046912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763863.044312:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049046912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763863.044324:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763863.044329:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763863.044334:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049046912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763863.044366:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763863.044399:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763863.044417:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763863/real 1789763863] req@ffff9811e1a23480 x1876700049047040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763863.044436:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049047040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763863.044449:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763863.044454:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763863.044521:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763863.044534:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763863.044568:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763863.044585:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763863/real 1789763863] req@ffff9811e1a23800 x1876700049047168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763863.044615:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049047168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763863.044627:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763863.044632:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763863.044636:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763863.044673:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763863.044709:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763863.044727:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763863/real 1789763863] req@ffff9811e1a23100 x1876700049047296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763863.044747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049047296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763863.044760:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763863.044765:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763863.044770:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763863.299563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763863.939694:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763864.323575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763864.963690:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763865.347530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763865.987583:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763866.371504:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763867.011702:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763867.395595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763868.035708:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763868.163657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763868.163673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763868.163680:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763868.163694:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3317 00000100:00080000:1.0:1789763868.163712:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763868.163757:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763868.163821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049047424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763868.163875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763868.163881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763868.163884:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763868.163890:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3317 00000100:00080000:1.0:1789763868.163898:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763868.163905:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763868.163920:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763868.163941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049047552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763868.163962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763868.163975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763868.163979:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049047424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763868.163981:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763868.164004:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3317 00000100:00080000:1.0:1789763868.164010:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763868.164016:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763868.164023:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763868.164042:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049047680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763868.164055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763868.164060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763868.164073:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763868.164079:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3317 00000100:00080000:1.0:1789763868.164089:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763868.164096:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763868.164105:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789763868.164106:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763868.164131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049047808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763868.164145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763868.164150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763868.164153:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763868.164163:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3317 00000100:00080000:1.0:1789763868.164170:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763868.164176:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763868.164183:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763868.164186:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763868/real 1789763868] req@ffff9811c92c6d80 x1876700049047424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763868.164207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049047936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763868.164244:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049047424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763868.164291:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763868.164297:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763868.164338:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049047424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763868.164389:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049047552:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763868.164439:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763868.164500:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763868/real 1789763868] req@ffff9811c92c7480 x1876700049047552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763868.164520:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049047552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763868.164532:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763868.164537:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763868.164541:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049047552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763868.164551:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763868.164581:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763868.164603:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763868/real 1789763868] req@ffff9811c92c4000 x1876700049047680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763868.164628:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049047680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763868.164645:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763868.164652:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763868.164659:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763868.164688:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763868.164734:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763868.164759:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763868/real 1789763868] req@ffff9811c92c5500 x1876700049047808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763868.164785:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049047808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763868.164798:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763868.164802:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763868.164806:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763868.164815:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763868.164845:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763868.164859:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763868/real 1789763868] req@ffff9811c92c4e00 x1876700049047936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763868.164873:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049047936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763868.164883:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763868.164887:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763868.164890:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049047936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763868.419548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763869.059739:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763869.443562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763870.083711:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763870.467525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763871.107732:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763871.491582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763872.131656:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763872.515518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763873.155584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763873.283579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763873.283588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763873.283592:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763873.283603:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3322 00000100:00080000:1.0:1789763873.283634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763873.283657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763873.283709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049048064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763873.283740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763873.283744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763873.283746:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763873.283750:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3322 00000100:00080000:1.0:1789763873.283754:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763873.283759:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763873.283770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763873.283789:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049048064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763873.283789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049048192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763873.283807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763873.283812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763873.283814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763873.283818:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3322 00000100:00080000:1.0:1789763873.283823:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763873.283828:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763873.283833:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763873.283843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049048320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763873.283852:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763873.283861:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763873.283863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763873.283865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763873.283889:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3322 00000100:00080000:1.0:1789763873.283897:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763873.283916:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763873.283926:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763873.283947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049048448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763873.283960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763873.283965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763873.283968:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763873.283977:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3322 00000100:00100000:0.0:1789763873.283980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763873/real 1789763873] req@ffff9811c92c7b80 x1876700049048064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763873.283982:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763873.283987:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763873.283992:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763873.284015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049048576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763873.284051:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049048064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763873.284070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763873.284076:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763873.284082:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049048064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763873.284120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049048192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763873.284164:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763873.284185:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763873/real 1789763873] req@ffff9811c92c5880 x1876700049048192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763873.284203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049048192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763873.284216:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763873.284220:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763873.284225:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049048192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763873.284233:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763873.284322:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763873.284336:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763873/real 1789763873] req@ffff9811c92c5180 x1876700049048320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763873.284345:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049048320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763873.284362:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763873.284386:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763873.284393:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763873.284408:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763873.284462:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763873.284549:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763873/real 1789763873] req@ffff9811c92c4e00 x1876700049048448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763873.284561:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049048448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763873.284569:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763873.284573:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763873.284577:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763873.284584:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763873.284607:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763873.284618:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763873/real 1789763873] req@ffff9811c92c5500 x1876700049048576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763873.284631:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049048576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763873.284640:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763873.284644:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763873.284657:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763873.539604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763874.179742:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763874.563525:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763875.203604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763875.587595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763876.227712:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763876.611557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763877.252662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763877.635607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763878.275666:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763878.403551:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763878.403562:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763878.403568:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763878.403600:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3327 00000100:00080000:1.0:1789763878.403614:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763878.403625:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763878.403681:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049048704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763878.403727:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763878.403732:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763878.403734:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763878.403739:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3327 00000100:00080000:1.0:1789763878.403745:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763878.403750:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763878.403763:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763878.403781:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049048832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763878.403808:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763878.403813:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763878.403815:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763878.403820:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3327 00000100:00080000:1.0:1789763878.403825:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763878.403836:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763878.403839:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049048704:192.168.203.113@tcp:250:kworker.0 00000008:00100000:1.0:1789763878.403843:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763878.403860:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049048960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763878.403921:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763878.403926:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763878.403928:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763878.403947:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3327 00000100:00080000:1.0:1789763878.403954:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763878.403961:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763878.403971:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763878.403986:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049049088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763878.403997:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763878.404003:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1789763878.404015:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763878.404043:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763878.404057:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3327 00000100:00080000:1.0:1789763878.404064:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763878.404072:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763878.404080:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763878.404111:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049049216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763878.404132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763878/real 1789763878] req@ffff9811c92c4000 x1876700049048704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763878.404161:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049048704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763878.404174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763878.404178:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763878.404183:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049048704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763878.404214:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049048832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763878.404271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763878.404293:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763878/real 1789763878] req@ffff9811c92c7480 x1876700049048832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763878.404303:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049048832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763878.404310:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763878.404312:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763878.404315:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049048832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763878.404319:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763878.404365:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763878.404373:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763878/real 1789763878] req@ffff9811c92c6d80 x1876700049048960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763878.404383:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049048960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763878.404389:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763878.404391:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763878.404394:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049048960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763878.404398:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763878.404409:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763878.404416:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763878/real 1789763878] req@ffff9811c92c5f80 x1876700049049088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763878.404433:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049049088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763878.404439:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763878.404442:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763878.404444:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763878.404451:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763878.404467:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763878.404561:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763878/real 1789763878] req@ffff9811c92c4a80 x1876700049049216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763878.404583:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049049216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763878.404589:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763878.404592:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763878.404594:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763878.659600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763879.299790:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763879.683579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763880.323705:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763880.707621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763881.347605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763881.731604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763882.371648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763882.755585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763883.395625:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763883.523588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763883.523600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763883.523619:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763883.523631:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3333 00000100:00080000:1.0:1789763883.523646:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763883.523656:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763883.523709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049049344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763883.523742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763883.523746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763883.523748:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763883.523752:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3333 00000100:00080000:1.0:1789763883.523756:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763883.523761:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763883.523773:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763883.523789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049049472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763883.523804:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049049344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763883.523805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763883.523813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763883.523826:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763883.523834:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3333 00000100:00080000:1.0:1789763883.523839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763883.523843:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763883.523848:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763883.523862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049049600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763883.523888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763883.523892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763883.523894:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763883.523898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3333 00000100:00080000:1.0:1789763883.523904:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763883.523907:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763883.523914:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763883.523922:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763883.523931:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049049728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763883.523944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763883.523947:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763883.523949:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763883.523954:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3333 00000100:00080000:1.0:1789763883.523958:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763883.523961:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763883.523965:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763883.523976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049049856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763883.524008:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763883/real 1789763883] req@ffff9811c92c4380 x1876700049049344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763883.524040:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049049344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763883.524053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763883.524057:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763883.524061:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049049344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763883.524090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049049472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763883.524118:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763883.524133:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763883/real 1789763883] req@ffff9811c92c6680 x1876700049049472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763883.524145:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049049472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763883.524169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763883.524173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763883.524176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049049472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763883.524183:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763883.524227:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763883.524239:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763883/real 1789763883] req@ffff9811c92c5c00 x1876700049049600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763883.524251:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049049600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763883.524260:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763883.524263:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763883.524266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763883.524271:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763883.524289:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763883.524298:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763883/real 1789763883] req@ffff9811c92c7800 x1876700049049728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763883.524309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049049728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763883.524316:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763883.524319:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763883.524330:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763883.524336:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763883.524356:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763883.524365:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763883/real 1789763883] req@ffff9811c92c4700 x1876700049049856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763883.524376:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049049856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763883.524383:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763883.524386:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763883.524389:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049049856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763883.779589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763884.419701:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763884.803584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763885.443661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763885.827601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763886.467729:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763886.851623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763887.491683:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763887.875601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763888.515745:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763888.643634:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763888.643668:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763888.643676:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763888.643696:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3338 00000100:00080000:0.0:1789763888.643715:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763888.643731:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763888.643818:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049049984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763888.643865:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763888.643871:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763888.643873:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763888.643879:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3338 00000100:00080000:0.0:1789763888.643885:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763888.643921:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763888.643936:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049049984:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763888.643938:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763888.643981:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049050112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763888.644002:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763888.644008:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763888.644011:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763888.644035:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3338 00000100:00080000:0.0:1789763888.644063:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763888.644071:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:1.0:1789763888.644081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789763888.644088:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763888.644124:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049050240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763888.644142:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763888.644147:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763888.644173:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763888.644183:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763888/real 1789763888] req@ffff9811c2e29c00 x1876700049049984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763888.644187:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3338 00000100:00080000:0.0:1789763888.644195:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763888.644208:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763888.644223:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763888.644247:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049050368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763888.644256:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049049984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763888.644267:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763888.644285:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763888.644289:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763888.644292:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763888.644300:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3338 00000100:00080000:0.0:1789763888.644311:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763888.644311:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763888.644328:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763888.644334:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049049984:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763888.644338:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763888.644360:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049050496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763888.644386:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049050112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763888.644430:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763888.644448:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763888/real 1789763888] req@ffff9811c2e2bb80 x1876700049050112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763888.644509:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049050112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763888.644518:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763888.644521:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763888.644524:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049050112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763888.644530:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763888.644551:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763888.644566:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763888/real 1789763888] req@ffff9811c2e2a680 x1876700049050240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763888.644603:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049050240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763888.644616:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763888.644620:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763888.644624:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763888.644630:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763888.644674:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763888.644697:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763888/real 1789763888] req@ffff9811c2e2b100 x1876700049050368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763888.644722:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049050368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763888.644739:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763888.644746:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763888.644753:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763888.644766:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763888.644806:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763888.644828:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763888/real 1789763888] req@ffff9811c2e2aa00 x1876700049050496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763888.644853:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049050496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763888.644894:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763888.644899:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763888.644904:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763888.899618:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763889.539712:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763889.923585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763890.563616:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763890.947576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763891.587647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763891.971611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763892.611789:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763892.995598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763893.635605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763893.763577:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763893.763587:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763893.763591:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763893.763601:0:488:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3343 00000100:00080000:3.0:1789763893.763614:0:488:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763893.763620:0:488:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763893.763662:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049050624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763893.763684:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763893.763687:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763893.763689:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763893.763691:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3343 00000100:00080000:3.0:1789763893.763693:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763893.763696:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763893.763704:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763893.763713:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049050752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763893.763722:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763893.763748:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049050624:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763893.763756:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763893.763758:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763893.763762:0:488:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3343 00000100:00080000:3.0:1789763893.763764:0:488:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763893.763768:0:488:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763893.763772:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763893.763782:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049050880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763893.763799:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763893.763802:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763893.763803:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763893.763806:0:488:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3343 00000100:00080000:3.0:1789763893.763809:0:488:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763893.763812:0:488:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763893.763815:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763893.763821:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049051008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763893.763827:0:488:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763893.763830:0:488:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763893.763831:0:488:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763893.763835:0:488:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3343 00000800:00000100:1.0:1789763893.763840:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763893.763840:0:488:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763893.763848:0:488:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763893.763860:0:488:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763893.763876:0:488:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049051136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763893.763887:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763893/real 1789763893] req@ffff9811e1a22680 x1876700049050624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763893.763903:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049050624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763893.763930:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763893.763934:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763893.763937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049050624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763893.763943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049050752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763893.763970:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763893.763980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763893/real 1789763893] req@ffff9811e1a21c00 x1876700049050752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763893.763988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049050752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763893.763994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763893.764004:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763893.764007:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049050752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763893.764040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:1.0:1789763893.764057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763893.764065:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763893/real 1789763893] req@ffff9811e1a21880 x1876700049050880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763893.764073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049050880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763893.764088:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763893.764090:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763893.764092:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049050880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:1.0:1789763893.764096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763893.764108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763893.764151:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763893/real 1789763893] req@ffff9811e1a22300 x1876700049051008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763893.764160:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049051008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763893.764165:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763893.764168:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763893.764170:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:1.0:1789763893.764174:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:1.0:1789763893.764195:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:1.0:1789763893.764202:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763893/real 1789763893] req@ffff9811e1a21180 x1876700049051136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763893.764209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049051136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763893.764214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763893.764216:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:1.0:1789763893.764218:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763894.019599:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763894.659772:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763895.043598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763895.683636:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763896.067544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763896.707728:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763897.091651:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763897.731616:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763898.115663:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763898.755766:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763898.883654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763898.883664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763898.883668:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763898.883681:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3348 00000100:00080000:1.0:1789763898.883694:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763898.883703:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763898.883750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049051264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763898.883786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763898.883790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763898.883792:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763898.883795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3348 00000100:00080000:1.0:1789763898.883798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763898.883802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763898.883836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763898.883861:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049051264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789763898.883861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700049051392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763898.883887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763898.883891:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763898.883893:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763898.883897:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3348 00000100:00080000:1.0:1789763898.883901:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763898.883906:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763898.883912:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763898.883924:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049051520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763898.883934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763898.883938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763898.883939:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763898.883942:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3348 00000100:00080000:1.0:1789763898.883946:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763898.883949:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789763898.883953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763898.883960:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763898.883972:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049051648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763898.883980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763898.883983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763898.883992:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763898.883997:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3348 00000100:00080000:1.0:1789763898.884002:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763898.884007:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763898/real 1789763898] req@ffff9811c92c6a00 x1876700049051264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763898.884009:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763898.884022:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763898.884036:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049051264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763898.884044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049051776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763898.884050:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763898.884062:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763898.884070:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049051264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763898.884140:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049051392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763898.884175:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763898.884193:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763898/real 1789763898] req@ffff9811c92c7100 x1876700049051392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763898.884209:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700049051392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763898.884233:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763898.884238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763898.884242:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049051392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763898.884250:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763898.884278:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763898.884291:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763898/real 1789763898] req@ffff9811c92c6300 x1876700049051520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763898.884305:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049051520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763898.884314:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763898.884318:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763898.884321:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763898.884328:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763898.884350:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763898.884361:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763898/real 1789763898] req@ffff9811c92c4700 x1876700049051648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763898.884393:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049051648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763898.884403:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763898.884406:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763898.884420:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763898.884428:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763898.884456:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763898.884518:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763898/real 1789763898] req@ffff9811c92c7800 x1876700049051776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763898.884536:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049051776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763898.884547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763898.884551:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763898.884566:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049051776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763899.139578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763899.779661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763900.163572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763900.803739:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763901.187595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763901.827598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763902.211645:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763902.851725:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763903.235669:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763903.875661:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763904.003711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763904.003735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763904.003775:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763904.003791:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3353 00000100:00080000:2.0:1789763904.003808:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763904.003818:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763904.003886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638700 x1876700049051904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763904.003923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763904.003930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763904.003933:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763904.003939:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3353 00000100:00080000:2.0:1789763904.003943:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763904.003948:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763904.003961:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763904.003977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638e00 x1876700049052032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763904.003993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763904.003998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763904.004001:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763904.004027:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3353 00000100:00080000:2.0:1789763904.004033:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763904.004067:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049051904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789763904.004074:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763904.004085:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763904.004111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638380 x1876700049052160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763904.004128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763904.004132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763904.004136:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763904.004142:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3353 00000100:00080000:2.0:1789763904.004146:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763904.004151:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763904.004158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763904.004178:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049052288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763904.004191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763904.004196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763904.004199:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763904.004208:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3353 00000800:00000100:0.0:1789763904.004210:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789763904.004220:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763904.004229:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763904.004236:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763904.004253:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049052416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763904.004286:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763904/real 1789763904] req@ffff9811d8638700 x1876700049051904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763904.004319:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638700 x1876700049051904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763904.004335:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763904.004340:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763904.004345:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049051904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763904.004355:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049052032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763904.004428:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763904.004547:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763904/real 1789763904] req@ffff9811d8638e00 x1876700049052032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763904.004567:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638e00 x1876700049052032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763904.004579:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763904.004583:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763904.004588:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049052032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763904.004640:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763904.004667:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763904.004685:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763904/real 1789763904] req@ffff9811d8638380 x1876700049052160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763904.004705:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638380 x1876700049052160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763904.004719:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763904.004725:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763904.004731:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763904.004743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763904.004791:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763904.004808:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763904/real 1789763904] req@ffff9811d8638000 x1876700049052288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763904.004824:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049052288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763904.004845:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763904.004850:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763904.004855:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763904.004863:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763904.004890:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763904.004902:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763904/real 1789763904] req@ffff9811d863aa00 x1876700049052416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763904.004935:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049052416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763904.004943:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763904.004947:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763904.004950:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763904.259580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763904.899669:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763905.283616:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763905.923811:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763906.307610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763906.947765:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763907.331610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763907.971650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763908.355644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763908.995736:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763909.123647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763909.123661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763909.123669:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763909.123687:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3358 00000100:00080000:2.0:1789763909.123705:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763909.123715:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763909.123782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049052544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763909.123828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763909.123837:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763909.123843:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763909.123851:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3358 00000100:00080000:2.0:1789763909.123857:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763909.123862:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763909.123876:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763909.123895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049052672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763909.123910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763909.123915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763909.123917:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763909.123926:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3358 00000100:00080000:2.0:1789763909.123930:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763909.123937:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763909.123971:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763909.123974:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049052544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763909.124029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049052800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763909.124065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763909.124074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763909.124079:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763909.124097:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3358 00000100:00080000:2.0:1789763909.124108:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763909.124116:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763909.124126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:3.0:1789763909.124138:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763909.124160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049052928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763909.124175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763909.124181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763909.124184:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763909.124194:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3358 00000100:00080000:2.0:1789763909.124199:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763909.124203:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789763909.124209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763909.124219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049053056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763909.124328:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763909/real 1789763909] req@ffff9811d863b480 x1876700049052544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763909.124377:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049052544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763909.124392:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763909.124397:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763909.124671:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049052544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763909.124758:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049052672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763909.124820:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763909.124842:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763909/real 1789763909] req@ffff9811d863a300 x1876700049052672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763909.124858:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049052672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763909.124870:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763909.124874:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763909.124878:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049052672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763909.124911:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:3.0:1789763909.124934:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763909.124947:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763909/real 1789763909] req@ffff9811d863b800 x1876700049052800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763909.124961:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049052800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763909.124989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763909.124994:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763909.124998:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763909.125032:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763909.125068:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763909.125091:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763909/real 1789763909] req@ffff9811d8638a80 x1876700049052928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763909.125116:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049052928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763909.125133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763909.125140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763909.125146:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049052928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763909.125159:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:3.0:1789763909.125194:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763909.125210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763909/real 1789763909] req@ffff9811d863a680 x1876700049053056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763909.125225:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049053056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763909.125235:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763909.125239:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:3.0:1789763909.125254:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763909.379587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763910.019796:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763910.403635:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763911.043728:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763911.427623:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763912.067709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763912.451601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763913.091778:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763913.475617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763914.115725:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763914.243639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763914.243653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763914.243659:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763914.243674:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3363 00000100:00080000:3.0:1789763914.243689:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763914.243702:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763914.243770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049053184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763914.243823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763914.243829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763914.243831:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763914.243837:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3363 00000100:00080000:3.0:1789763914.243843:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763914.243849:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763914.243863:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763914.243882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049053312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763914.243927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763914.243933:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763914.243936:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763914.243942:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3363 00000100:00080000:3.0:1789763914.243949:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763914.243958:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763914.243990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049053184:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763914.243994:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763914.244035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049053440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763914.244053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763914.244059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763914.244061:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763914.244067:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3363 00000100:00080000:3.0:1789763914.244073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763914.244079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763914.244087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763914.244101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049053568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763914.244115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763914.244119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763914.244123:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763914.244134:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3363 00000800:00000100:0.0:1789763914.244138:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763914.244159:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763914.244169:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763914.244183:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763914.244203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049053696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763914.244219:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763914/real 1789763914] req@ffff9811e1a22a00 x1876700049053184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763914.244258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049053184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763914.244298:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763914.244305:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763914.244311:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049053184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763914.244324:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049053312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763914.244359:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763914.244375:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763914/real 1789763914] req@ffff9811e1a22d80 x1876700049053312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763914.244392:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049053312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763914.244402:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763914.244407:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763914.244411:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049053312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763914.244445:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763914.244516:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763914.244533:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763914/real 1789763914] req@ffff9811e1a21180 x1876700049053440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763914.244549:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049053440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763914.244559:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763914.244564:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763914.244568:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763914.244589:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763914.244620:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763914.244634:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763914/real 1789763914] req@ffff9811e1a22300 x1876700049053568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763914.244648:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049053568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763914.244659:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763914.244662:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763914.244666:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763914.244673:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763914.244693:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763914.244704:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763914/real 1789763914] req@ffff9811e1a21880 x1876700049053696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763914.244718:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049053696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763914.244727:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763914.244730:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763914.244734:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049053696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763914.499577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763915.139699:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763915.523605:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763916.163728:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763916.547625:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763917.187754:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763917.571586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763918.211734:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763918.595603:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763919.235619:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763919.363614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763919.363627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763919.363633:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763919.363646:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3368 00000100:00080000:3.0:1789763919.363661:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763919.363672:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763919.363740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049053824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763919.363791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763919.363796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763919.363799:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763919.363804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3368 00000100:00080000:3.0:1789763919.363809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763919.363829:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763919.363854:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763919.363873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049053952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763919.363891:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763919.363901:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049053824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763919.363902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763919.363908:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763919.363918:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3368 00000100:00080000:3.0:1789763919.363924:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763919.363981:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763919.363989:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763919.364005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049054080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763919.364035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763919.364040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763919.364043:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763919.364050:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3368 00000100:00080000:3.0:1789763919.364056:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763919.364061:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763919.364068:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763919.364087:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763919.364089:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049054208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763919.364106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763919.364120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763919.364123:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763919.364129:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3368 00000100:00080000:3.0:1789763919.364136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763919.364143:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763919.364152:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763919.364154:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763919/real 1789763919] req@ffff9811e1a21c00 x1876700049053824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763919.364174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049054336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763919.364190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049053824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763919.364224:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763919.364230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763919.364237:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049053824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763919.364270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049053952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763919.364313:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763919.364326:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763919/real 1789763919] req@ffff9811e1a22680 x1876700049053952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763919.364338:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049053952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763919.364374:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763919.364377:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763919.364381:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049053952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763919.364387:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763919.364405:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763919.364415:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763919/real 1789763919] req@ffff9811e1a23100 x1876700049054080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763919.364459:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049054080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763919.364467:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763919.364470:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763919.364473:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763919.364479:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763919.364498:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763919.364525:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763919/real 1789763919] req@ffff9811e1a23800 x1876700049054208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763919.364538:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049054208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763919.364546:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763919.364548:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763919.364552:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763919.364557:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763919.364682:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763919.364698:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763919/real 1789763919] req@ffff9811e1a23480 x1876700049054336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763919.364713:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049054336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763919.364755:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763919.364759:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763919.364765:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763919.619621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763920.259670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763920.643596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763921.283726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763921.667578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763922.307658:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763922.691596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763923.331696:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763923.715644:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763924.355644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763924.483607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763924.483622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763924.483628:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763924.483642:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3374 00000100:00080000:1.0:1789763924.483658:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763924.483673:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789763924.483756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049054464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763924.483814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763924.483819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763924.483822:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763924.483828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3374 00000100:00080000:1.0:1789763924.483834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763924.483839:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763924.483853:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763924.483871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049054592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763924.483970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763924.483975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763924.483987:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763924.483994:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3374 00000100:00080000:1.0:1789763924.483999:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763924.484005:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049054464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789763924.484010:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763924.484026:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763924.484045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049054720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763924.484063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763924.484068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763924.484070:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763924.484075:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3374 00000100:00080000:1.0:1789763924.484079:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763924.484084:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763924.484090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763924.484101:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049054848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763924.484112:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763924.484116:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763924.484118:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763924.484123:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3374 00000100:00080000:1.0:1789763924.484136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763924.484142:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789763924.484149:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789763924.484166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049054976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763924.484182:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049054592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763924.484235:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763924.484270:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763924.484303:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763924.739592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763925.379665:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763925.763637:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763926.403604:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763926.787574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763927.427689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763927.555975:0:2346:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789763927.556037:0:2346:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763927.556061:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763927.556119:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763927.556132:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763927.556143:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789763927.556157:0:2346:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763927.556245:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763924/real 1789763927] req@ffff9811c92c4380 x1876700049054720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763927.556297:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049054720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763927.556317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763927.556325:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763927.556332:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763927.556346:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763924/real 1789763927] req@ffff9811c92c4a80 x1876700049054848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763927.556370:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049054848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763927.556409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763927.556417:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763927.556424:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763927.556437:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763924/real 1789763927] req@ffff9811c92c5f80 x1876700049054976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763927.556533:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049054976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763927.556547:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763927.556553:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763927.556558:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049054976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763927.556568:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763924/real 1789763927] req@ffff9811c92c5c00 x1876700049054464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763927.556587:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049054464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763927.556599:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763927.556603:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763927.556608:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049054464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763927.556618:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763924/real 1789763927] req@ffff9811c92c6680 x1876700049054592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763927.556637:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049054592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763927.556664:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763927.556671:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763927.556677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049054592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763927.811584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763928.451643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763928.835584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763929.475651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763929.603689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763929.603701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763929.603707:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763929.603722:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3379 00000100:00080000:3.0:1789763929.603737:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763929.603747:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763929.603797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049055104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763929.603841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763929.603846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763929.603849:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763929.603876:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3379 00000100:00080000:3.0:1789763929.603882:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763929.603887:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763929.603901:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763929.603942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049055232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763929.603966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763929.603973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763929.603978:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763929.603989:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3379 00000100:00100000:0.0:1789763929.603992:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049055104:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763929.603999:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763929.604034:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763929.604046:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763929.604076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049055360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763929.604098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763929.604106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763929.604110:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763929.604127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3379 00000800:00000100:0.0:1789763929.604135:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763929.604138:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763929.604152:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763929.604164:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763929.604192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049055488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763929.604213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763929.604220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763929.604227:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763929.604231:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763929/real 1789763929] req@ffff9811e1a21500 x1876700049055104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763929.604245:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3379 00000100:00080000:3.0:1789763929.604260:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763929.604275:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763929.604287:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763929.604292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049055104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763929.604317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049055616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763929.604320:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763929.604341:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763929.604356:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049055104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763929.604372:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049055232:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763929.604425:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763929.604451:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763929/real 1789763929] req@ffff9811e1a20700 x1876700049055232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763929.604574:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049055232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763929.604594:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763929.604601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763929.604608:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049055232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763929.604668:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055360:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763929.604715:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763929.604740:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763929/real 1789763929] req@ffff9811e1a20a80 x1876700049055360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763929.604764:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049055360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763929.604782:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763929.604788:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763929.604795:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763929.604808:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055488:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763929.604877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763929.604904:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763929/real 1789763929] req@ffff9811e1a21f80 x1876700049055488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763929.604931:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049055488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763929.604948:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763929.604954:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763929.604961:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763929.604975:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055616:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763929.605016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763929.605043:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763929/real 1789763929] req@ffff9811e1a20000 x1876700049055616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763929.605092:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049055616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763929.605109:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763929.605115:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763929.605122:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049055616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763929.859584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763930.499655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763930.883577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763931.523676:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763931.907585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763932.547669:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763932.931558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763933.571845:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763933.955539:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763934.595691:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763934.723623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763934.723638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763934.723644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763934.723659:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3384 00000100:00080000:3.0:1789763934.723675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763934.723686:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763934.723752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049055744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763934.723795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763934.723801:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763934.723803:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763934.723808:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3384 00000100:00080000:3.0:1789763934.723834:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763934.723841:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763934.723869:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763934.723898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049055872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763934.723937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763934.723939:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049055744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763934.723952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763934.723962:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763934.723997:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3384 00000100:00080000:3.0:1789763934.724005:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763934.724011:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763934.724021:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763934.724043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049056000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763934.724058:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763934.724072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763934.724078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763934.724080:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763934.724087:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3384 00000100:00080000:3.0:1789763934.724093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763934.724105:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763934.724111:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763934.724127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049056128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763934.724140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763934.724148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763934/real 1789763934] req@ffff9811e1a20e00 x1876700049055744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763934.724149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763934.724157:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763934.724183:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3384 00000100:00080000:3.0:1789763934.724189:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763934.724195:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763934.724202:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763934.724203:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049055744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763934.724223:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049056256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763934.724248:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.724259:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763934.724266:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049055744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763934.724304:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049055872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763934.724344:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763934.724360:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763934/real 1789763934] req@ffff9811e1a20380 x1876700049055872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.724376:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049055872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.724408:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.724413:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763934.724418:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049055872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.724426:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763934.724452:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763934.724525:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763934/real 1789763934] req@ffff9811e1a20000 x1876700049056000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.724542:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049056000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.724553:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.724557:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763934.724561:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.724568:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763934.724594:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763934.724613:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763934/real 1789763934] req@ffff9811e1a21f80 x1876700049056128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.724627:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049056128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.724636:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.724639:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763934.724654:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763934.724662:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763934.724697:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763934.724710:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763934/real 1789763934] req@ffff9811e1a20a80 x1876700049056256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.724725:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049056256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.724734:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.724738:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763934.724741:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763934.979535:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763935.619791:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763936.003573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763936.643692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763937.027589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763937.667677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763938.051571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763938.691707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763939.075611:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763939.715768:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763939.843579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763939.843591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763939.843597:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763939.843611:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3389 00000100:00080000:3.0:1789763939.843641:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763939.843653:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763939.843717:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049056384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763939.843752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763939.843757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763939.843760:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763939.843764:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3389 00000100:00080000:3.0:1789763939.843769:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763939.843773:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763939.843806:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763939.843822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049056512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763939.843876:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049056384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763939.843881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763939.843894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763939.843897:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763939.843904:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3389 00000100:00080000:3.0:1789763939.843910:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763939.843915:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763939.843923:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763939.843941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049056640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763939.843954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763939.843959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763939.843962:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763939.843966:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3389 00000100:00080000:3.0:1789763939.843971:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763939.843975:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763939.843982:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763939.843994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049056768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763939.844009:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763939.844016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763939.844022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763939.844025:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763939.844032:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3389 00000100:00080000:3.0:1789763939.844046:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763939.844053:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763939.844060:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763939.844102:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049056896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763939.844103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763939/real 1789763939] req@ffff9811e1a20700 x1876700049056384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763939.844167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049056384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763939.844190:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763939.844200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763939.844208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049056384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763939.844253:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049056512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763939.844304:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763939.844350:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763939/real 1789763939] req@ffff9811e1a21500 x1876700049056512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763939.844380:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049056512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763939.844398:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763939.844405:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763939.844413:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049056512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763939.844426:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763939.844525:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763939.844553:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763939/real 1789763939] req@ffff9811e1a23480 x1876700049056640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763939.844578:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049056640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763939.844595:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763939.844601:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763939.844608:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763939.844622:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763939.844668:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763939.844693:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763939/real 1789763939] req@ffff9811e1a23800 x1876700049056768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763939.844721:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049056768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763939.844753:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763939.844760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763939.844768:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763939.844810:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763939.844858:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763939.844882:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763939/real 1789763939] req@ffff9811e1a23100 x1876700049056896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763939.844908:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049056896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763939.844925:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763939.844931:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763939.844937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049056896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763940.099593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763940.739708:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763941.123619:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763941.763703:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763942.147588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763942.787659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763943.171586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763943.811687:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763944.195636:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763944.835700:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763944.963643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763944.963659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763944.963666:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763944.963681:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3394 00000100:00080000:3.0:1789763944.963697:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763944.963710:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763944.963798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049057024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763944.963845:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763944.963853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763944.963856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763944.963881:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3394 00000100:00080000:3.0:1789763944.963896:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763944.963908:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763944.963916:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049057024:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763944.963963:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763944.963994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049057152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763944.964022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763944.964033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1789763944.964035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763944.964040:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763944.964050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3394 00000100:00080000:3.0:1789763944.964061:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763944.964074:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763944.964082:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763944.964110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049057280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763944.964137:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763944/real 1789763944] req@ffff9811e1a22680 x1876700049057024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763944.964180:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049057024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763944.964196:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763944.964202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763944.964208:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049057024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763944.964264:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049057152:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763944.964281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763944.964292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763944.964297:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763944.964308:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3394 00000800:00000100:0.0:1789763944.964326:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763944.964356:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763944/real 1789763944] req@ffff9811e1a21c00 x1876700049057152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763944.964382:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049057152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763944.964395:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763944.964399:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763944.964404:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049057152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763944.964412:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763944.964444:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763944.964518:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763944/real 1789763944] req@ffff9811e1a21880 x1876700049057280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763944.964556:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049057280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763944.964566:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763944.964583:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763944.964585:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763944.964596:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763944.964602:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763944.964614:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763944.964627:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049057408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763944.964774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763944.964780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763944.964796:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763944.964806:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3394 00000100:00080000:3.0:1789763944.964815:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763944.964816:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057408:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763944.964830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763944.964846:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763944.964874:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763944.964884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049057536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763944.964906:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763944/real 1789763944] req@ffff9811e1a22300 x1876700049057408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763944.964941:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049057408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763944.964954:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763944.964959:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763944.964964:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763944.964983:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763944.965010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763944.965023:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763944/real 1789763944] req@ffff9811e1a21180 x1876700049057536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763944.965037:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049057536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763944.965047:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763944.965051:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763944.965054:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763945.219697:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763945.859686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763946.243598:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763946.883686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763947.267590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763947.907708:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763948.291569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763948.931663:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763949.315576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763949.955662:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763950.083515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763950.083525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763950.083529:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763950.083540:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3399 00000100:00080000:3.0:1789763950.083551:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763950.083558:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763950.083599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049057664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763950.083623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763950.083629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763950.083631:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763950.083636:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3399 00000100:00080000:3.0:1789763950.083654:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763950.083661:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763950.083672:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763950.083673:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049057664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763950.083687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049057792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763950.083714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763950.083720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763950.083722:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763950.083726:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3399 00000100:00080000:3.0:1789763950.083731:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763950.083735:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763950.083740:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763950.083753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049057920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763950.083760:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763950.083765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763950.083768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763950.083770:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763950.083774:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3399 00000100:00080000:3.0:1789763950.083777:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763950.083782:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763950.083787:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763950.083797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049058048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763950.083804:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763950/real 1789763950] req@ffff9811e1a22d80 x1876700049057664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763950.083822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763950.083826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789763950.083827:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049057664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763950.083829:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763950.083834:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3399 00000100:00080000:3.0:1789763950.083838:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763950.083843:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763950.083845:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763950.083849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789763950.083850:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763950.083855:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049057664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763950.083860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049058176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763950.083882:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049057792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763950.083912:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763950.083925:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763950/real 1789763950] req@ffff9811e1a22a00 x1876700049057792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763950.083936:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049057792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763950.083943:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763950.083946:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763950.083948:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049057792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763950.083953:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763950.083967:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763950.083974:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763950/real 1789763950] req@ffff9811e1a21180 x1876700049057920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763950.083983:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049057920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763950.083989:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763950.083991:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763950.083993:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049057920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763950.084004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763950.084020:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763950.084027:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763950/real 1789763950] req@ffff9811e1a22300 x1876700049058048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763950.084036:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049058048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763950.084042:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763950.084044:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763950.084046:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763950.084051:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763950.084062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763950.084069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763950/real 1789763950] req@ffff9811e1a21880 x1876700049058176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763950.084087:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049058176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763950.084096:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763950.084098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763950.084101:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763950.339590:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789763950.979602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763951.363547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763952.003726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763952.387621:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763953.027651:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763953.411666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763954.051661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763954.435572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763955.075662:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763955.203647:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763955.203690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763955.203695:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763955.203725:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3404 00000100:00080000:0.0:1789763955.203738:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763955.203745:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763955.203787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049058304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763955.203818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763955.203822:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763955.203824:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763955.203828:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3404 00000100:00080000:0.0:1789763955.203831:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763955.203835:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763955.203845:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763955.203857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049058432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763955.203868:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049058304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763955.203870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763955.203879:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763955.203881:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763955.203885:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3404 00000100:00080000:0.0:1789763955.203887:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763955.203891:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763955.203895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763955.203906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049058560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763955.203922:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763955.203926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763955.203928:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763955.203932:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3404 00000100:00080000:0.0:1789763955.203937:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763955.203941:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763955.203946:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789763955.203956:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763955.203961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049058688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763955.203971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763955.203975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763955.203977:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763955.203980:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3404 00000100:00080000:0.0:1789763955.203984:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763955.203987:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763955.203990:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763955.204000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049058816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763955.204012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763955/real 1789763955] req@ffff9811c2e28380 x1876700049058304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763955.204045:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049058304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763955.204056:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763955.204060:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763955.204064:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049058304:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763955.204088:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049058432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763955.204109:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763955.204120:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763955/real 1789763955] req@ffff9811c2e28a80 x1876700049058432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763955.204157:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049058432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763955.204169:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763955.204173:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763955.204178:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049058432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763955.204187:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763955.204215:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763955.204226:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763955/real 1789763955] req@ffff9811c2e29f80 x1876700049058560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763955.204237:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049058560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763955.204244:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763955.204247:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763955.204250:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763955.204255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763955.204271:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763955.204278:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763955/real 1789763955] req@ffff9811c2e2b800 x1876700049058688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763955.204288:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049058688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763955.204322:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763955.204326:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763955.204329:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763955.204334:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763955.204354:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763955.204363:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763955/real 1789763955] req@ffff9811c2e2b480 x1876700049058816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763955.204374:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049058816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763955.204381:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763955.204383:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763955.204386:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049058816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763955.459591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763956.099781:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763956.483595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763957.123718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763957.507606:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763958.147730:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763958.531600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763959.171765:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763959.555624:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763960.195723:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763960.323654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763960.323669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763960.323719:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763960.323738:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3409 00000100:00080000:0.0:1789763960.323758:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763960.323772:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763960.323832:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700049058944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763960.323875:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763960.323883:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763960.323887:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763960.323895:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3409 00000100:00080000:0.0:1789763960.323902:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763960.323911:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763960.323928:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763960.323954:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049058944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763960.323968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700049059072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763960.323990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763960.323997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763960.324001:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763960.324008:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3409 00000100:00080000:0.0:1789763960.324014:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763960.324021:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763960.324029:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763960.324047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700049059200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763960.324076:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789763960.324089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763960.324095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763960.324098:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763960.324122:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3409 00000100:00080000:0.0:1789763960.324131:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763960.324139:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763960/real 1789763960] req@ffff9811c2e2a300 x1876700049058944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763960.324149:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763960.324170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049059328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763960.324183:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700049058944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763960.324189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763960.324197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763960.324200:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763960.324203:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763960.324206:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3409 00000100:00080000:2.0:1789763960.324211:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789763960.324212:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763960.324219:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049058944:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789763960.324227:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763960.324243:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049059456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763960.324291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049059072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763960.324343:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324372:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763960/real 1789763960] req@ffff9811c2e28000 x1876700049059072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763960.324394:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700049059072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763960.324405:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763960.324409:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763960.324414:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049059072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763960.324422:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059200:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763960.324452:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324511:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763960/real 1789763960] req@ffff9811c2e29500 x1876700049059200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763960.324528:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700049059200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763960.324538:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763960.324542:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763960.324546:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763960.324554:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763960.324580:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324592:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763960/real 1789763960] req@ffff9811c2e2b480 x1876700049059328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763960.324619:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049059328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763960.324660:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763960.324665:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763960.324677:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763960.324686:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763960.324718:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763960.324733:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763960/real 1789763960] req@ffff9811c2e2b800 x1876700049059456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763960.324747:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049059456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763960.324757:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763960.324761:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763960.324765:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763960.579655:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763961.219723:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763961.603608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763962.243726:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763962.627602:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763963.267744:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763963.651646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763964.291692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763964.675642:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763965.315758:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763965.443634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763965.443676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763965.443681:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763965.443694:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3415 00000100:00080000:0.0:1789763965.443708:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763965.443718:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763965.443790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049059584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763965.443835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763965.443841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763965.443844:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763965.443850:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3415 00000100:00080000:0.0:1789763965.443858:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763965.443867:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763965.443883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763965.443902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049059712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763965.443939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763965.443945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763965.443945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049059584:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763965.443952:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763965.443974:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3415 00000100:00080000:0.0:1789763965.443978:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763965.443982:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763965.444009:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763965.444029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049059840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763965.444052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763965.444056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763965.444058:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763965.444065:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3415 00000100:00080000:0.0:1789763965.444073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763965.444083:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789763965.444100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:0.0:1789763965.444121:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763965.444140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049059968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763965.444152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763965.444155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763965.444157:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763965.444165:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3415 00000100:00080000:0.0:1789763965.444170:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763965.444175:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763965.444182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789763965.444197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763965/real 1789763965] req@ffff9811c2e29f80 x1876700049059584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763965.444201:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049060096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763965.444326:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049059584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763965.444375:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763965.444390:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763965.444396:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049059584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763965.444443:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049059712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763965.444540:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763965.444561:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763965/real 1789763965] req@ffff9811c2e28a80 x1876700049059712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763965.444578:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049059712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763965.444616:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763965.444621:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763965.444626:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049059712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763965.444644:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763965.444670:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763965.444684:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763965/real 1789763965] req@ffff9811c2e28380 x1876700049059840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763965.444699:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049059840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763965.444716:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763965.444720:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763965.444724:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763965.444743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763965.444770:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763965.444784:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763965/real 1789763965] req@ffff9811c2e2aa00 x1876700049059968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763965.444799:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049059968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763965.444809:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763965.444812:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763965.444816:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049059968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763965.444823:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763965.444843:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763965.444854:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763965/real 1789763965] req@ffff9811c2e2b100 x1876700049060096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763965.444868:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049060096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763965.444877:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763965.444880:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763965.444884:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763965.699567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763966.339706:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763966.723620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763967.363708:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763967.747607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763968.387694:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763968.771711:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763969.411707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763969.795609:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763970.435791:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763970.563747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763970.563757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763970.563761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763970.563772:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3420 00000100:00080000:3.0:1789763970.563784:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763970.563790:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763970.563833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049060224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763970.563861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763970.563864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763970.563865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763970.563868:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3420 00000100:00080000:3.0:1789763970.563870:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763970.563879:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763970.563887:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763970.563896:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049060352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763970.563904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763970.563908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763970.563924:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763970.563933:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3420 00000100:00080000:3.0:1789763970.563936:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763970.563940:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763970.563945:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763970.563955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049060480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763970.563966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763970.563970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763970.563973:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763970.563985:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3420 00000100:00100000:2.0:1789763970.563988:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049060224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763970.563992:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763970.564009:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763970.564018:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763970.564033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049060608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763970.564042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763970.564045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763970.564046:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763970.564049:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3420 00000100:00080000:3.0:1789763970.564052:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763970.564056:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763970.564066:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763970.564084:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049060736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763970.564127:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763970.564202:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763970/real 1789763970] req@ffff9811e1a21c00 x1876700049060224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763970.564258:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049060224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763970.564281:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763970.564290:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763970.564298:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049060224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763970.564314:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049060352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763970.564361:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763970.564387:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763970/real 1789763970] req@ffff9811e1a22680 x1876700049060352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763970.564413:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049060352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763970.564430:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763970.564437:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763970.564445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049060352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763970.564546:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763970.564615:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763970.564636:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763970/real 1789763970] req@ffff9811e1a23100 x1876700049060480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763970.564659:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049060480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763970.564702:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763970.564709:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763970.564717:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763970.564729:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763970.564769:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763970.564789:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763970/real 1789763970] req@ffff9811e1a23800 x1876700049060608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763970.564813:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049060608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763970.564830:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763970.564836:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763970.564843:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763970.564856:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763970.564897:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763970.564921:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763970/real 1789763970] req@ffff9811e1a23480 x1876700049060736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763970.564947:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049060736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763970.564964:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763970.564971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763970.564991:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049060736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763970.819587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763971.459682:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763971.843579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763972.483718:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763972.867646:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763973.507692:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763973.891604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763974.531773:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763974.915575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763975.555686:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763975.683614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763975.683623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763975.683627:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763975.683637:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3425 00000100:00080000:0.0:1789763975.683648:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763975.683656:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763975.683696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049060864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763975.683720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763975.683724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763975.683726:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763975.683730:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3425 00000100:00080000:0.0:1789763975.683734:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763975.683738:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763975.683748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763975.683764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049060992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763975.683778:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763975.683781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763975.683783:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763975.683794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3425 00000100:00080000:0.0:1789763975.683813:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763975.683817:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763975.683823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763975.683836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049061120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763975.683845:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049060864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763975.683849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763975.683867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763975.683869:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763975.683874:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3425 00000100:00080000:0.0:1789763975.683876:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763975.683879:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763975.683883:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763975.683895:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049061248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763975.683902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763975.683905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763975.683907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763975.683910:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3425 00000100:00080000:0.0:1789763975.683913:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763975.683915:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763975.683919:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763975.683928:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049061376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763975.683960:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763975.684022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763975/real 1789763975] req@ffff9811c2e2a680 x1876700049060864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763975.684049:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049060864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763975.684065:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763975.684071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763975.684077:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049060864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789763975.684087:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049060992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763975.684126:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763975.684152:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763975/real 1789763975] req@ffff9811c2e2bb80 x1876700049060992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763975.684181:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049060992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763975.684201:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763975.684207:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763975.684214:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049060992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763975.684255:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789763975.684288:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763975.684323:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763975/real 1789763975] req@ffff9811c2e29c00 x1876700049061120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763975.684340:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049061120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763975.684351:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763975.684355:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763975.684359:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763975.684367:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763975.684390:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763975.684401:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763975/real 1789763975] req@ffff9811c2e28700 x1876700049061248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763975.684416:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049061248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763975.684425:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763975.684429:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763975.684432:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763975.684439:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789763975.684457:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789763975.684562:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763975/real 1789763975] req@ffff9811c2e29180 x1876700049061376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763975.684579:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049061376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763975.684602:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763975.684607:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789763975.684611:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763975.939573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763976.579692:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763976.963587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763977.603718:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763977.987548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763978.627668:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763979.011607:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763979.651623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763980.035579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763980.675617:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763980.803587:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763980.803595:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763980.803599:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763980.803609:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3430 00000100:00080000:3.0:1789763980.803619:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763980.803627:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763980.803663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049061504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763980.803688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763980.803692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763980.803694:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763980.803698:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3430 00000100:00080000:3.0:1789763980.803717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763980.803724:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763980.803726:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049061504:192.168.203.113@tcp:250:kworker.0 00000008:00100000:3.0:1789763980.803734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763980.803750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049061632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763980.803760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763980.803763:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763980.803765:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763980.803768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3430 00000800:00000100:0.0:1789763980.803793:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763980.803817:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763980.803848:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763980/real 1789763980] req@ffff9811e1a21500 x1876700049061504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763980.803867:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049061504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763980.803876:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763980.803880:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763980.803882:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049061504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763980.803903:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049061632:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763980.803904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763980.803928:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763980.803931:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763980.803952:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763980/real 1789763980] req@ffff9811e1a20700 x1876700049061632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763980.803967:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049061632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763980.803974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763980.803976:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763980.803979:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049061632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763980.804205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049061760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763980.804253:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061760:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763980.804256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763980.804260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763980.804262:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763980.804266:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3430 00000100:00080000:3.0:1789763980.804273:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763980.804278:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763980.804284:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763980.804284:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763980.804302:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049061888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763980.804311:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763980/real 1789763980] req@ffff9811e1a20a80 x1876700049061760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763980.804348:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049061760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763980.804357:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763980.804362:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763980.804366:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763980.804399:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061888:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763980.804408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763980.804418:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763980.804427:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763980/real 1789763980] req@ffff9811e1a21f80 x1876700049061888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763980.804435:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049061888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763980.804441:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763980.804443:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763980.804445:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049061888:192.168.203.113@tcp:8:kworker.0 00000100:00080000:3.0:1789763980.804450:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763980.804567:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763980.804571:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3430 00000100:00080000:3.0:1789763980.804575:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763980.804578:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763980.804588:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789763980.804599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049062016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763980.804655:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763980.804692:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763980.804702:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763980/real 1789763980] req@ffff9811e1a20000 x1876700049062016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763980.804711:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049062016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763980.804723:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763980.804725:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763980.804728:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763981.059626:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763981.699684:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763982.083589:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763982.723686:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763983.107531:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763983.747602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763984.131565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763984.771597:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763985.155567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763985.795609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763985.923560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763985.923577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763985.923580:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763985.923591:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3435 00000100:00080000:3.0:1789763985.923601:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763985.923610:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763985.923654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049062144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763985.923675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763985.923693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763985.923694:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763985.923698:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3435 00000100:00080000:3.0:1789763985.923702:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763985.923706:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763985.923716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763985.923728:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049062272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763985.923730:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049062144:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763985.923747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763985.923749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763985.923751:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763985.923754:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3435 00000100:00080000:3.0:1789763985.923756:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763985.923759:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763985.923762:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763985.923771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049062400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763985.923777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763985.923780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763985.923781:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763985.923784:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3435 00000100:00080000:3.0:1789763985.923788:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763985.923791:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763985.923803:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789763985.923813:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789763985.923818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049062528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763985.923831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763985.923835:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763985.923837:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763985.923842:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3435 00000100:00080000:3.0:1789763985.923846:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763985.923850:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763985.923855:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763985.923863:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763985/real 1789763985] req@ffff9811e1a20380 x1876700049062144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763985.923867:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049062656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763985.923901:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049062144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763985.923914:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763985.923918:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763985.923922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049062144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763985.923948:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049062272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763985.923969:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763985.923982:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763985/real 1789763985] req@ffff9811e1a20e00 x1876700049062272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763985.923995:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049062272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763985.924003:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763985.924007:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763985.924019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049062272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763985.924027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763985.924048:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763985.924058:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763985/real 1789763985] req@ffff9811e1a20000 x1876700049062400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763985.924070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049062400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763985.924078:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763985.924082:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763985.924085:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763985.924091:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763985.924108:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763985.924118:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763985/real 1789763985] req@ffff9811e1a21f80 x1876700049062528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763985.924129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049062528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763985.924137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763985.924140:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763985.924143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763985.924148:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763985.924162:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763985.924179:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763985/real 1789763985] req@ffff9811e1a20a80 x1876700049062656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763985.924190:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049062656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763985.924197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763985.924200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763985.924204:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049062656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763986.179575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763986.819649:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763987.203563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763987.843643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763988.227571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763988.867639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763989.251566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763989.891641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763990.275572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763990.915651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763991.043630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763991.043640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763991.043644:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763991.043655:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3440 00000100:00080000:0.0:1789763991.043667:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763991.043674:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789763991.043714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049062784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763991.043740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763991.043743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763991.043744:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763991.043747:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3440 00000100:00080000:0.0:1789763991.043749:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763991.043752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763991.043777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763991.043787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049062912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763991.043797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763991.043800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763991.043802:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763991.043805:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3440 00000100:00080000:0.0:1789763991.043808:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763991.043810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763991.043813:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763991.043826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049063040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763991.043829:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049062784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789763991.043854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763991.043857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763991.043858:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763991.043863:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3440 00000100:00080000:0.0:1789763991.043864:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763991.043868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763991.043871:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763991.043881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049063168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763991.043888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763991.043889:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763991.043895:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763991.043898:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3440 00000100:00080000:0.0:1789763991.043902:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763991.043906:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789763991.043910:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763991.043921:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049063296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763991.043964:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049062912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763991.044002:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789763991.044030:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789763991.044059:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763991.299566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763991.939573:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763992.323584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763992.963588:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763993.347541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763993.987655:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763994.115747:0:2348:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:2.0:1789763994.115794:0:2348:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763994.115814:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763994.115863:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763994.115875:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763994.115884:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:2.0:1789763994.115895:0:2348:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789763994.115951:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763991/real 1789763994] req@ffff9811c2e29880 x1876700049063040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763994.115990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049063040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763994.116423:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763994.116431:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763994.116437:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763994.116448:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763991/real 1789763994] req@ffff9811c2e29180 x1876700049063168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763994.116513:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049063168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763994.116525:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763994.116529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763994.116532:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763994.116566:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763991/real 1789763994] req@ffff9811c2e28700 x1876700049063296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763994.116583:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049063296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763994.116590:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763994.116595:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763994.116598:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763994.116606:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763991/real 1789763994] req@ffff9811c2e2ad80 x1876700049062784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763994.116618:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049062784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763994.116625:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763994.116628:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763994.116631:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049062784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789763994.116638:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763991/real 1789763994] req@ffff9811c2e28e00 x1876700049062912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763994.116649:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049062912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763994.116658:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763994.116661:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763994.116665:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049062912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789763994.371692:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763995.011625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763995.395579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763996.035595:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763996.163602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763996.163613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763996.163618:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763996.163629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3445 00000100:00080000:3.0:1789763996.163655:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763996.163663:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789763996.163701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049063424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763996.163776:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049063424:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789763996.163805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763996.163811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763996.163814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763996.163820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3445 00000100:00080000:3.0:1789763996.163826:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763996.163831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:0.0:1789763996.163852:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:3.0:1789763996.163887:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763996.163900:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763996/real 1789763996] req@ffff9811e1a20700 x1876700049063424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763996.163908:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049063552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763996.163928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763996.163935:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049063424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763996.163935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763996.163938:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763996.163948:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3445 00000100:00080000:3.0:1789763996.163956:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763996.163957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763996.163963:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00080000:0.0:1789763996.163965:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000008:00100000:3.0:1789763996.163972:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763996.163973:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049063424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789763996.163995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049063680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763996.164008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763996.164010:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049063552:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789763996.164021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763996.164024:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763996.164030:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3445 00000100:00080000:3.0:1789763996.164037:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763996.164042:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763996.164046:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763996.164054:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763996.164062:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763996/real 1789763996] req@ffff9811e1a21500 x1876700049063552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763996.164071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049063808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763996.164081:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049063552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763996.164086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763996.164097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763996.164099:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763996.164100:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763996.164108:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789763996.164110:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3445 00000100:00100000:0.0:1789763996.164115:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049063552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763996.164128:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063680:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789763996.164156:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789763996.164174:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763996.164178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763996/real 1789763996] req@ffff9811e1a23480 x1876700049063680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763996.164184:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789763996.164196:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789763996.164202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049063680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763996.164214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763996.164215:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049063936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763996.164221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763996.164226:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789763996.164232:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063808:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763996.164255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763996.164268:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763996/real 1789763996] req@ffff9811e1a23800 x1876700049063808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763996.164283:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049063808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763996.164303:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763996.164307:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763996.164310:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789763996.164325:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063936:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789763996.164343:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789763996.164354:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763996/real 1789763996] req@ffff9811e1a23100 x1876700049063936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763996.164380:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049063936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763996.164395:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763996.164403:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789763996.164408:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049063936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789763996.419572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789763997.059590:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763997.443516:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789763998.083668:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763998.467508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789763999.107601:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763999.491514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764000.131638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764000.515518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764001.155650:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764001.283592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764001.283601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764001.283605:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764001.283614:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3450 00000100:00080000:0.0:1789764001.283625:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764001.283631:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764001.283670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049064064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764001.283696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764001.283699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764001.283700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764001.283703:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3450 00000100:00080000:0.0:1789764001.283705:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764001.283708:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764001.283716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764001.283725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049064192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764001.283734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764001.283743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049064064:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789764001.283752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764001.283754:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764001.283758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3450 00000100:00080000:0.0:1789764001.283760:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764001.283764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764001.283767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764001.283778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049064320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764001.283785:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764001.283788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764001.283789:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764001.283791:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3450 00000100:00080000:0.0:1789764001.283802:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764001.283804:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764001.283808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789764001.283813:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764001.283818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049064448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764001.283825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764001.283828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764001.283829:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764001.283832:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3450 00000100:00080000:0.0:1789764001.283837:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764001.283840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764001.283848:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764001.283857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049064576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764001.283863:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764001/real 1789764001] req@ffff9811c2e28e00 x1876700049064064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764001.283896:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049064064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764001.283917:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764001.283921:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764001.283931:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049064064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764001.283959:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049064192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764001.283994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764001.284012:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764001/real 1789764001] req@ffff9811c2e2ad80 x1876700049064192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764001.284022:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049064192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764001.284034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764001.284036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764001.284039:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049064192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764001.284043:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764001.284057:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764001.284081:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764001/real 1789764001] req@ffff9811c2e28700 x1876700049064320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764001.284090:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049064320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764001.284096:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764001.284098:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764001.284106:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764001.284116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764001.284132:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764001.284150:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764001/real 1789764001] req@ffff9811c2e29180 x1876700049064448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764001.284159:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049064448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764001.284164:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764001.284166:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764001.284176:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764001.284180:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764001.284201:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764001.284216:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764001/real 1789764001] req@ffff9811c2e29880 x1876700049064576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764001.284224:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049064576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764001.284236:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764001.284238:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764001.284240:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764001.539514:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764002.179629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764002.563534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764003.203600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764003.587547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764004.227638:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764004.611523:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764005.251604:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764005.635566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764006.275623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764006.403585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764006.403594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764006.403599:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764006.403625:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3455 00000100:00080000:2.0:1789764006.403635:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764006.403642:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764006.403676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049064704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764006.403717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764006.403722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764006.403724:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764006.403730:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3455 00000100:00080000:2.0:1789764006.403737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764006.403740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764006.403750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764006.403762:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049064704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764006.403763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049064832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764006.403787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764006.403790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764006.403792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764006.403804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3455 00000100:00080000:2.0:1789764006.403807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764006.403810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764006.403814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764006.403824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049064960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764006.403852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764006.403855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764006.403856:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764006.403860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3455 00000100:00080000:2.0:1789764006.403864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764006.403865:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789764006.403870:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764006.403875:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764006.403891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049065088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764006.403897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764006.403912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764006.403913:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764006.403915:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3455 00000100:00080000:2.0:1789764006.403921:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764006.403925:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764006.403934:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764006.403935:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764006/real 1789764006] req@ffff9811d8639500 x1876700049064704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764006.403948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049065216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764006.403960:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049064704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764006.403971:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764006.403976:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764006.403980:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049064704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764006.404004:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049064832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764006.404022:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764006.404030:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764006/real 1789764006] req@ffff9811d8639880 x1876700049064832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764006.404039:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049064832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764006.404045:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764006.404047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764006.404063:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049064832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764006.404068:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764006.404082:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764006.404089:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764006/real 1789764006] req@ffff9811d863b100 x1876700049064960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764006.404097:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049064960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764006.404102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764006.404105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764006.404107:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049064960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764006.404111:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764006.404123:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764006.404128:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764006/real 1789764006] req@ffff9811d8639f80 x1876700049065088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764006.404135:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049065088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764006.404140:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764006.404142:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764006.404144:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764006.404147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764006.404156:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764006.404169:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764006/real 1789764006] req@ffff9811d863ad80 x1876700049065216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764006.404177:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049065216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764006.404182:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764006.404185:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764006.404187:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764006.659579:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764007.300558:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764007.683564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764008.323609:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764008.707534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764009.347573:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764009.731584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764010.371598:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764010.755528:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764011.395600:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764011.523586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764011.523597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764011.523601:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764011.523610:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3461 00000100:00080000:2.0:1789764011.523638:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764011.523645:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764011.523679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639180 x1876700049065344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764011.523704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764011.523707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764011.523708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764011.523711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3461 00000100:00080000:2.0:1789764011.523719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764011.523722:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764011.523729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764011.523742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863bb80 x1876700049065472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764011.523756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764011.523761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789764011.523763:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049065344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789764011.523763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764011.523774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3461 00000100:00080000:2.0:1789764011.523779:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764011.523784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764011.523789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764011.523801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639c00 x1876700049065600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764011.523809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764011.523817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764011.523818:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764011.523822:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3461 00000100:00080000:2.0:1789764011.523825:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764011.523829:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:2.0:1789764011.523831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764011.523838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764011.523849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863ad80 x1876700049065728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764011.523856:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764011.523859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764011.523860:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764011.523863:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3461 00000100:00080000:2.0:1789764011.523866:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764011.523869:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764011.523874:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764011/real 1789764011] req@ffff9811d8639180 x1876700049065344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789764011.523874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764011.523886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639f80 x1876700049065856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764011.523910:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639180 x1876700049065344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764011.523923:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764011.523929:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764011.523933:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049065344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764011.523957:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049065472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764011.523976:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764011.523985:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764011/real 1789764011] req@ffff9811d863bb80 x1876700049065472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764011.524006:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863bb80 x1876700049065472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764011.524018:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764011.524021:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764011.524023:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049065472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764011.524027:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764011.524043:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764011.524050:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764011/real 1789764011] req@ffff9811d8639c00 x1876700049065600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764011.524058:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639c00 x1876700049065600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764011.524063:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764011.524064:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764011.524066:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764011.524070:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764011.524081:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764011.524087:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764011/real 1789764011] req@ffff9811d863ad80 x1876700049065728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764011.524094:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863ad80 x1876700049065728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764011.524108:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764011.524110:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764011.524112:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764011.524116:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764011.524130:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764011.524136:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764011/real 1789764011] req@ffff9811d8639f80 x1876700049065856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764011.524143:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639f80 x1876700049065856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764011.524148:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764011.524149:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764011.524151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049065856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764011.779629:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764012.419580:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764012.803536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764013.443661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764013.827572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764014.467679:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764014.851570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764015.491790:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764015.875592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764016.515631:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764016.643537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764016.643547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764016.643552:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764016.643564:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3466 00000100:00080000:2.0:1789764016.643576:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764016.643585:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764016.643626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b100 x1876700049065984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764016.643679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764016.643684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764016.643686:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764016.643691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3466 00000100:00080000:2.0:1789764016.643696:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764016.643701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764016.643713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764016.643725:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049065984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764016.643737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639880 x1876700049066112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764016.643751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764016.643755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764016.643765:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764016.643771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3466 00000100:00080000:2.0:1789764016.643780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764016.643785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764016.643791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764016.643805:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764016.643808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8639500 x1876700049066240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764016.643822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764016.643826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764016.643828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764016.643833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3466 00000100:00080000:2.0:1789764016.643845:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764016.643851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764016.643855:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764016/real 1789764016] req@ffff9811d863b100 x1876700049065984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789764016.643868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764016.643882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a680 x1876700049066368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764016.643885:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b100 x1876700049065984/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764016.643891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764016.643899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764016.643902:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764016.643909:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3466 00000100:00080000:2.0:1789764016.643915:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764016.643917:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764016.643918:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764016.643925:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1789764016.643926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764016.643932:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049065984:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764016.643940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638a80 x1876700049066496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764016.643965:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049066112:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764016.643994:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764016.644006:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764016/real 1789764016] req@ffff9811d8639880 x1876700049066112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764016.644019:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639880 x1876700049066112/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764016.644028:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764016.644033:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764016.644039:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049066112:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764016.644047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066240:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764016.644083:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764016.644102:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764016/real 1789764016] req@ffff9811d8639500 x1876700049066240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764016.644117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8639500 x1876700049066240/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764016.644125:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764016.644130:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764016.644134:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066240:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764016.644153:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066368:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764016.644178:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764016.644191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764016/real 1789764016] req@ffff9811d863a680 x1876700049066368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764016.644202:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a680 x1876700049066368/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764016.644211:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764016.644227:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764016.644231:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066368:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764016.644238:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066496:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764016.644260:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764016.644273:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764016/real 1789764016] req@ffff9811d8638a80 x1876700049066496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764016.644285:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638a80 x1876700049066496/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764016.644293:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764016.644296:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764016.644299:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066496:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764016.899594:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764017.539613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764017.923581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764018.563632:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764018.947565:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764019.587633:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764019.971567:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764020.611596:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764020.995614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764021.635613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764021.763556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764021.763585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764021.763589:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764021.763599:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3471 00000100:00080000:0.0:1789764021.763611:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764021.763619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764021.763660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049066624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764021.763695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764021.763700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764021.763702:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764021.763706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3471 00000100:00080000:0.0:1789764021.763710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764021.763714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764021.763724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764021.763741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049066624:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764021.763742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049066752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764021.763760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764021.763764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764021.763766:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764021.763771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3471 00000100:00080000:0.0:1789764021.763775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764021.763779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764021.763785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764021.763797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049066880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764021.763813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764021.763817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764021.763819:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764021.763824:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789764021.763829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3471 00000100:00080000:0.0:1789764021.763848:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764021.763853:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764021.763860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764021.763867:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764021/real 1789764021] req@ffff9811c2e29c00 x1876700049066624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764021.763876:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049067008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764021.763888:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049066624/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764021.763889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764021.763899:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764021.763903:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764021.763912:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3471 00000100:00080000:2.0:1789764021.763914:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764021.763917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764021.763922:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00080000:0.0:1789764021.763926:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764021.763932:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049066624:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789764021.763933:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764021.763947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049067136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764021.763968:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049066752:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764021.763991:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764021.764001:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764021/real 1789764021] req@ffff9811c2e2bb80 x1876700049066752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764021.764010:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049066752/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764021.764016:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764021.764018:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764021.764020:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049066752:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764021.764024:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066880:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764021.764035:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764021.764041:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764021/real 1789764021] req@ffff9811c2e2a680 x1876700049066880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764021.764048:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049066880/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764021.764053:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764021.764055:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764021.764057:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049066880:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764021.764062:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067008:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764021.764084:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764021.764095:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764021/real 1789764021] req@ffff9811c2e2b100 x1876700049067008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764021.764111:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049067008/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764021.764119:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764021.764122:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764021.764126:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067008:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764021.764132:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067136:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764021.764154:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764021.764167:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764021/real 1789764021] req@ffff9811c2e2aa00 x1876700049067136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764021.764191:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049067136/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764021.764198:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764021.764201:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764021.764203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067136:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764022.019666:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764022.659629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764023.043575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764023.683655:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764024.067553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764024.707588:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764025.091568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764025.731612:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764026.115560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764026.755622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764026.883576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764026.883585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764026.883589:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764026.883599:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3476 00000100:00080000:3.0:1789764026.883610:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764026.883617:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764026.883662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049067264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764026.883702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764026.883706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764026.883708:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764026.883713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3476 00000100:00080000:3.0:1789764026.883716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764026.883720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764026.883730:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764026.883743:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049067392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764026.883751:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049067264:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764026.883756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764026.883761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764026.883763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764026.883768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3476 00000100:00080000:3.0:1789764026.883771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764026.883775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764026.883780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764026.883797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049067520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764026.883806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764026.883811:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764026.883813:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764026.883817:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3476 00000800:00000100:2.0:1789764026.883826:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764026.883828:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764026.883835:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764026.883841:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764026.883851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049067648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764026.883860:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764026.883863:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764026.883865:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764026.883870:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764026/real 1789764026] req@ffff9811e1a22680 x1876700049067264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764026.883871:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3476 00000100:00080000:3.0:1789764026.883876:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764026.883883:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764026.883889:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764026.883897:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049067264/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764026.883902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049067776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764026.883927:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764026.883933:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764026.883937:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049067264:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764026.883958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049067392:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764026.883978:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764026.883989:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764026/real 1789764026] req@ffff9811e1a21c00 x1876700049067392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764026.884001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049067392/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764026.884009:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764026.884011:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764026.884014:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049067392:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764026.884020:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067520:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764026.884044:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764026.884054:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764026/real 1789764026] req@ffff9811e1a21880 x1876700049067520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764026.884065:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049067520/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764026.884072:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764026.884075:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764026.884078:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067520:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764026.884088:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067648:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764026.884105:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764026.884113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764026/real 1789764026] req@ffff9811e1a22300 x1876700049067648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764026.884124:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049067648/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764026.884132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764026.884135:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764026.884138:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067648:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764026.884156:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067776:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764026.884173:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764026.884182:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764026/real 1789764026] req@ffff9811e1a21180 x1876700049067776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764026.884192:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049067776/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764026.884199:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764026.884202:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764026.884211:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049067776:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764027.139575:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764027.779578:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764028.163620:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764028.803653:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764029.187586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764029.827602:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764030.211548:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764030.851699:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764031.235563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764031.875648:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764032.003600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764032.003610:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764032.003615:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764032.003624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3481 00000100:00080000:1.0:1789764032.003634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764032.003640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764032.003675:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049067904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764032.003696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764032.003699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764032.003700:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764032.003704:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3481 00000100:00080000:1.0:1789764032.003707:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764032.003711:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764032.003721:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764032.003735:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049068032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764032.003743:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049067904:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789764032.003760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764032.003769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764032.003771:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764032.003775:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3481 00000100:00080000:1.0:1789764032.003778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764032.003782:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764032.003797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764032.003807:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049068160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764032.003814:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764032.003816:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764032.003817:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764032.003820:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3481 00000100:00080000:1.0:1789764032.003823:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764032.003828:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000800:00000100:2.0:1789764032.003831:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000008:00100000:1.0:1789764032.003833:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764032.003842:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049068288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764032.003849:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764032.003852:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764032.003853:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764032.003855:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3481 00000100:00080000:1.0:1789764032.003866:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764032.003868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764032.003873:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764032.003884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049068416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764032.003895:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764032/real 1789764032] req@ffff9811c92c6d80 x1876700049067904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764032.003924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049067904/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764032.003932:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764032.003935:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764032.003938:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049067904:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764032.003958:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049068032:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764032.003977:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764032.003985:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764032/real 1789764032] req@ffff9811c92c7480 x1876700049068032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764032.003993:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049068032/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764032.003998:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764032.004000:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764032.004002:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049068032:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764032.004006:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068160:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764032.004016:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764032.004021:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764032/real 1789764032] req@ffff9811c92c4000 x1876700049068160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764032.004027:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049068160/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764032.004040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764032.004042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764032.004044:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068160:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764032.004048:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068288:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764032.004061:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764032.004067:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764032/real 1789764032] req@ffff9811c92c5500 x1876700049068288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764032.004073:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049068288/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764032.004077:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764032.004079:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764032.004081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068288:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764032.004084:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068416:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764032.004093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764032.004098:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764032/real 1789764032] req@ffff9811c92c4e00 x1876700049068416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764032.004117:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049068416/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764032.004122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764032.004124:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764032.004131:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068416:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764032.259586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764032.899627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764033.283558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764033.923683:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764034.307601:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764034.947607:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764035.331600:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764035.971674:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764036.355557:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764036.995623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764037.123583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764037.123593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764037.123596:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764037.123605:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3486 00000100:00080000:1.0:1789764037.123616:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764037.123623:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764037.123657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049068544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764037.123678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764037.123681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764037.123682:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764037.123685:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3486 00000100:00080000:1.0:1789764037.123687:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764037.123689:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764037.123697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764037.123704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049068672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764037.123711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764037.123714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764037.123716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764037.123719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3486 00000100:00080000:1.0:1789764037.123721:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764037.123723:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764037.123726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764037.123741:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049068544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789764037.123757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049068800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764037.123765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764037.123767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764037.123768:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764037.123771:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3486 00000100:00080000:1.0:1789764037.123773:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764037.123776:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764037.123779:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764037.123790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049068928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764037.123798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789764037.123803:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789764037.123804:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764037.123808:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764037.123813:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3486 00000100:00080000:1.0:1789764037.123818:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764037.123822:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764037.123828:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764037.123840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049069056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764037.123844:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764037/real 1789764037] req@ffff9811c92c5180 x1876700049068544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764037.123866:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049068544/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764037.123874:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764037.123876:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764037.123879:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049068544:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764037.123884:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049068672:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764037.123897:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764037.123904:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764037/real 1789764037] req@ffff9811c92c5880 x1876700049068672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764037.123911:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049068672/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764037.123916:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764037.123917:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764037.123919:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049068672:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764037.123945:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068800:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764037.123957:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764037.123963:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764037/real 1789764037] req@ffff9811c92c7b80 x1876700049068800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764037.123970:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049068800/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764037.123975:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764037.123977:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764037.123979:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068800:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764037.123981:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068928:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764037.123990:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764037.123995:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764037/real 1789764037] req@ffff9811c92c4e00 x1876700049068928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764037.124001:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049068928/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764037.124005:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764037.124007:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764037.124009:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049068928:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764037.124011:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069056:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764037.124019:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764037.124024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764037/real 1789764037] req@ffff9811c92c5500 x1876700049069056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764037.124035:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049069056/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764037.124040:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764037.124042:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764037.124043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069056:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764037.379571:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764038.019691:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764038.403604:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764039.043623:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764039.427583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764040.067595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764040.451566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764041.091656:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764041.475558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764042.115625:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764042.243619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764042.243640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764042.243645:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764042.243656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3491 00000100:00080000:3.0:1789764042.243669:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764042.243677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764042.243720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049069184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764042.243743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764042.243746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764042.243748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764042.243751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3491 00000100:00080000:3.0:1789764042.243755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764042.243759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764042.243769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764042.243783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049069312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764042.243795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764042.243799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764042.243802:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764042.243822:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049069184:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764042.243824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3491 00000100:00080000:3.0:1789764042.243835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764042.243841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764042.243846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764042.243858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049069440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764042.243867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764042.243869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764042.243870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764042.243873:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3491 00000100:00080000:3.0:1789764042.243875:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764042.243877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764042.243880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764042.243887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049069568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764042.243892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764042.243894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764042.243896:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764042.243898:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3491 00000100:00080000:3.0:1789764042.243900:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764042.243902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764042.243906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789764042.243913:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789764042.243920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049069696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764042.243972:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764042/real 1789764042] req@ffff9811e1a22a00 x1876700049069184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764042.243994:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049069184/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764042.244008:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764042.244014:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764042.244019:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049069184:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764042.244040:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049069312:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764042.244069:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764042.244082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764042/real 1789764042] req@ffff9811e1a22d80 x1876700049069312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764042.244102:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049069312/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764042.244127:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764042.244130:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764042.244133:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049069312:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764042.244154:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069440:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764042.244170:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764042.244178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764042/real 1789764042] req@ffff9811e1a21180 x1876700049069440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764042.244185:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049069440/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764042.244191:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764042.244193:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764042.244195:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069440:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764042.244198:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069568:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764042.244209:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764042.244214:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764042/real 1789764042] req@ffff9811e1a22300 x1876700049069568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764042.244221:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049069568/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764042.244225:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764042.244227:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764042.244240:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069568:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764042.244244:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069696:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764042.244260:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764042.244269:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764042/real 1789764042] req@ffff9811e1a21880 x1876700049069696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764042.244287:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049069696/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764042.244295:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764042.244298:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764042.244301:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049069696:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764042.499564:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764043.139605:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764043.523566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764044.163577:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764044.547592:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764045.187626:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764045.571661:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764046.211613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764046.595559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764047.235597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764047.363521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764047.363528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764047.363531:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764047.363540:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3496 00000100:00080000:3.0:1789764047.363550:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764047.363557:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764047.363593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049069824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764047.363616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764047.363619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764047.363620:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764047.363623:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3496 00000100:00080000:3.0:1789764047.363625:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764047.363627:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764047.363634:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764047.363642:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049069952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764047.363667:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049069824:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764047.363676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764047.363681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764047.363683:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764047.363688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3496 00000100:00080000:3.0:1789764047.363692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764047.363697:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764047.363702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764047.363715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049070080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764047.363725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764047.363728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764047.363730:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764047.363734:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764047.363736:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3496 00000100:00080000:3.0:1789764047.363742:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764047.363747:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764047.363751:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764047.363760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049070208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764047.363766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764047.363768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764047.363769:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764047.363772:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3496 00000100:00080000:3.0:1789764047.363780:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764047.363781:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764047/real 1789764047] req@ffff9811e1a21c00 x1876700049069824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764047.363785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764047.363791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764047.363803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049070336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764047.363809:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049069824/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764047.363821:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764047.363825:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764047.363829:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049069824:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764047.363855:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049069952:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764047.363877:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764047.363897:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764047/real 1789764047] req@ffff9811e1a22680 x1876700049069952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764047.363909:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049069952/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764047.363916:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764047.363919:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764047.363922:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049069952:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764047.363927:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070080:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764047.363944:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764047.363954:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764047/real 1789764047] req@ffff9811e1a23100 x1876700049070080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764047.363964:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049070080/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764047.363971:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764047.363974:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764047.363977:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070080:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764047.363981:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070208:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764047.364012:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764047.364022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764047/real 1789764047] req@ffff9811e1a23800 x1876700049070208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764047.364032:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049070208/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764047.364044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764047.364047:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764047.364050:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070208:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764047.364055:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070336:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764047.364074:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764047.364084:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764047/real 1789764047] req@ffff9811e1a23480 x1876700049070336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764047.364095:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049070336/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764047.364102:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764047.364105:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764047.364108:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070336:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764047.619530:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764048.259595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764048.643544:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764049.283584:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764049.667534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764050.307653:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764050.692744:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764051.331602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764051.715570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764052.355685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764052.483662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764052.483669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764052.483673:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764052.483682:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3502 00000100:00080000:3.0:1789764052.483691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764052.483698:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764052.483730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049070464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764052.483754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764052.483757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764052.483758:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764052.483761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3502 00000100:00080000:3.0:1789764052.483763:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764052.483764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764052.483789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764052.483805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049070592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764052.483815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764052.483819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764052.483825:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764052.483827:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049070464:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764052.483831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3502 00000100:00080000:3.0:1789764052.483848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764052.483853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764052.483861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764052.483871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049070720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764052.483877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764052.483880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764052.483881:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764052.483885:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3502 00000100:00080000:3.0:1789764052.483899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764052.483902:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764052.483906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764052.483914:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049070848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764052.483915:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764052.483986:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764052/real 1789764052] req@ffff9811e1a21500 x1876700049070464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764052.484009:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764052.484034:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049070464/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764052.484048:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764052.484052:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764052.484056:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049070464:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764052.484096:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049070592:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789764052.484122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1789764052.484125:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764052.484131:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764052.484136:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3502 00000100:00080000:3.0:1789764052.484141:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764052.484148:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764052/real 1789764052] req@ffff9811e1a20700 x1876700049070592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764052.484150:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764052.484166:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764052.484175:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049070592/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764052.484184:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764052.484187:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764052.484189:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049070592:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764052.484195:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:3.0:1789764052.484224:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049070976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764052.484231:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764052.484271:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764052/real 1789764052] req@ffff9811e1a20a80 x1876700049070720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764052.484288:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049070720/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764052.484296:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764052.484300:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764052.484304:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070720:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764052.484311:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070848:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764052.484330:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764052.484340:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764052/real 1789764052] req@ffff9811e1a21f80 x1876700049070848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764052.484360:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049070848/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764052.484368:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764052.484371:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764052.484374:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070848:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764052.484391:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070976:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764052.484407:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764052.484415:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764052/real 1789764052] req@ffff9811e1a20000 x1876700049070976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764052.484423:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049070976/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764052.484431:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764052.484434:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764052.484437:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049070976:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764052.739584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764053.379680:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764053.763541:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764054.403614:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764054.787562:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764055.427602:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764055.811583:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764056.451645:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764056.835586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764057.475590:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764057.603693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764057.603703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764057.603708:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764057.603734:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3507 00000100:00080000:1.0:1789764057.603745:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764057.603752:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764057.603785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049071104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764057.603819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764057.603823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764057.603825:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764057.603828:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3507 00000100:00080000:1.0:1789764057.603832:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764057.603836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764057.603845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764057.603856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049071232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764057.603866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764057.603869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764057.603870:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764057.603873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3507 00000100:00080000:1.0:1789764057.603876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764057.603880:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764057.603884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764057.603893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049071360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764057.603912:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764057.605990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049071104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764057.606072:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049071232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764057.606090:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071360:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789764057.606156:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764057.606158:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764057.606162:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3507 00000100:00080000:1.0:1789764057.606166:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764057.606170:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764057.606176:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764057.606190:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049071488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764057.606232:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071488:192.168.203.113@tcp:8:kworker.0 00000100:00080000:1.0:1789764057.606272:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764057.606275:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764057.606277:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764057.606279:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3507 00000100:00080000:1.0:1789764057.606282:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764057.606285:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764057.606288:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764057.606297:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049071616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764057.606325:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764057.859574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764058.499721:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764058.883572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764059.523622:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764059.907570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764060.547710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789764060.676106:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789764060.676158:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789764060.676174:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764060.676208:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764060.676218:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764060.676229:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764060.676253:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:2.0:1789764060.676265:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764057/real 1789764060] req@ffff9811c92c6680 x1876700049071488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764060.676300:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049071488/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764060.676313:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764060.676318:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764060.676322:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071488:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764060.676330:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764057/real 1789764060] req@ffff9811c92c4000 x1876700049071104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764060.676343:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049071104/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764060.676352:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764060.676355:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764060.676359:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049071104:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764060.676366:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764057/real 1789764060] req@ffff9811c92c7480 x1876700049071232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764060.676378:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049071232/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764060.676386:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764060.676390:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764060.676393:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049071232:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764060.676421:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764057/real 1789764060] req@ffff9811c92c6d80 x1876700049071360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764060.676434:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049071360/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764060.676442:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764060.676446:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764060.676449:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071360:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764060.676505:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764057/real 1789764060] req@ffff9811c92c5c00 x1876700049071616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764060.676518:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049071616/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764060.676526:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764060.676529:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764060.676532:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049071616:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764060.931578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764061.571629:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764061.955536:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764062.595629:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764062.723546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764062.723557:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764062.723562:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764062.723571:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3512 00000100:00080000:1.0:1789764062.723592:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764062.723599:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764062.723642:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049071744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764062.723664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764062.723666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764062.723667:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764062.723670:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3512 00000100:00080000:1.0:1789764062.723672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764062.723675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764062.723685:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764062.723699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049071872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764062.723712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764062.723718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789764062.723720:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049071744:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789764062.723721:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764062.723736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3512 00000100:00080000:1.0:1789764062.723741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764062.723745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764062.723753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764062.723769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049072000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764062.723781:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764062.723790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764062.723792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764062.723796:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3512 00000100:00080000:1.0:1789764062.723803:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789764062.723804:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789764062.723815:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764062.723823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764062.723841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049072128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764062.723852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764062.723855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764062.723857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764062.723863:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3512 00000100:00100000:2.0:1789764062.723868:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764062/real 1789764062] req@ffff9811c92c5f80 x1876700049071744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764062.723870:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764062.723890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764062.723900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764062.723917:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049072256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764062.723922:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049071744/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764062.723961:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764062.723982:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764062.723987:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049071744:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764062.724019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049071872:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764062.724047:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764062.724061:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764062/real 1789764062] req@ffff9811c92c4a80 x1876700049071872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764062.724070:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049071872/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764062.724076:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764062.724078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764062.724080:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049071872:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764062.724085:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072000:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764062.724101:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764062.724126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764062/real 1789764062] req@ffff9811c92c4380 x1876700049072000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764062.724138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049072000/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764062.724145:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764062.724148:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764062.724151:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072000:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764062.724158:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072128:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764062.724186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764062.724197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764062/real 1789764062] req@ffff9811c92c7800 x1876700049072128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764062.724210:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049072128/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764062.724218:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764062.724221:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764062.724225:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072128:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764062.724231:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072256:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764062.724251:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764062.724261:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764062/real 1789764062] req@ffff9811c92c4700 x1876700049072256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764062.724272:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049072256/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764062.724279:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764062.724282:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764062.724286:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072256:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764062.979572:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764063.620534:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764064.003509:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764064.643626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764065.027518:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764065.667643:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764066.051580:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764066.691966:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764067.075522:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764067.715707:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764067.844961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764067.844971:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764067.844975:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764067.844985:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3517 00000100:00080000:1.0:1789764067.845012:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764067.845019:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764067.845056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6300 x1876700049072384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764067.845089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764067.845092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764067.845094:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764067.845098:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3517 00000100:00080000:1.0:1789764067.845101:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764067.845104:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764067.845113:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764067.845125:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7100 x1876700049072512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764067.845137:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764067.845140:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764067.845142:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764067.845146:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3517 00000100:00080000:1.0:1789764067.845149:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764067.845153:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764067.845158:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764067.845170:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6a00 x1876700049072640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764067.845176:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049072384:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789764067.845183:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764067.845193:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764067.845195:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764067.845200:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3517 00000100:00080000:1.0:1789764067.845205:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764067.845210:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764067.845216:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764067.845229:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4700 x1876700049072768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764067.845239:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764067.845242:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764067.845243:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764067.845246:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3517 00000100:00080000:1.0:1789764067.845250:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764067.845254:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764067.845259:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764067.845268:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7800 x1876700049072896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764067.845275:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764067.845349:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764067/real 1789764067] req@ffff9811c92c6300 x1876700049072384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.845390:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6300 x1876700049072384/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.845402:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.845407:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764067.845411:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049072384:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764067.845419:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049072512:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764067.845444:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764067.845456:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764067/real 1789764067] req@ffff9811c92c7100 x1876700049072512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.845510:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7100 x1876700049072512/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.845518:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.845521:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764067.845531:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049072512:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764067.845562:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072640:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764067.845585:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764067.845596:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764067/real 1789764067] req@ffff9811c92c6a00 x1876700049072640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.845609:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6a00 x1876700049072640/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.845628:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.845632:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764067.845636:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072640:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764067.845643:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072768:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764067.845666:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764067.845677:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764067/real 1789764067] req@ffff9811c92c4700 x1876700049072768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.845689:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4700 x1876700049072768/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.845697:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.845700:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764067.845703:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072768:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764067.845710:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072896:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764067.845728:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764067.845738:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764067/real 1789764067] req@ffff9811c92c7800 x1876700049072896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.845750:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7800 x1876700049072896/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.845757:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.845760:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764067.845769:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049072896:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764068.099526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764068.739727:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764069.123508:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764069.763854:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764070.147552:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764070.787722:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764071.171582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764071.811861:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764072.195576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764072.835944:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764072.963574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764072.963583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764072.963587:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764072.963598:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3522 00000100:00080000:1.0:1789764072.963610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764072.963642:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764072.963684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4380 x1876700049073024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764072.963830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764072.963834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764072.963836:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764072.963840:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3522 00000100:00080000:1.0:1789764072.963845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764072.963848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764072.963858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764072.963871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4a80 x1876700049073152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764072.963879:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049073024:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789764072.963887:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764072.963891:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764072.963893:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764072.963897:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3522 00000100:00080000:1.0:1789764072.963901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764072.963905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764072.963910:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764072.963921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5f80 x1876700049073280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764072.963930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764072.963934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764072.963942:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764072.963948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3522 00000800:00000100:2.0:1789764072.963951:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789764072.963952:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764072.963959:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764072.963964:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764072.963974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5c00 x1876700049073408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764072.963984:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764072.963987:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764072.963989:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764072.963993:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3522 00000100:00080000:1.0:1789764072.963997:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764072.964000:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764072/real 1789764072] req@ffff9811c92c4380 x1876700049073024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764072.964003:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764072.964010:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764072.964023:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6d80 x1876700049073536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764072.964030:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4380 x1876700049073024/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764072.964044:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764072.964048:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764072.964053:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049073024:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764072.964081:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049073152:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764072.964104:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764072.964117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764072/real 1789764072] req@ffff9811c92c4a80 x1876700049073152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764072.964129:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4a80 x1876700049073152/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764072.964137:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764072.964141:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764072.964144:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049073152:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764072.964150:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073280:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764072.964169:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764072.964187:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764072/real 1789764072] req@ffff9811c92c5f80 x1876700049073280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764072.964199:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5f80 x1876700049073280/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764072.964207:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764072.964210:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764072.964213:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073280:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764072.964218:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073408:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764072.964238:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764072.964249:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764072/real 1789764072] req@ffff9811c92c5c00 x1876700049073408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764072.964354:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5c00 x1876700049073408/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764072.964361:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764072.964364:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764072.964367:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073408:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764072.964373:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073536:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764072.964393:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764072.964403:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764072/real 1789764072] req@ffff9811c92c6d80 x1876700049073536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764072.964414:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6d80 x1876700049073536/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764072.964429:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764072.964433:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764072.964436:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073536:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764073.219582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764073.859715:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764074.243573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764074.883697:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764075.267631:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764075.907822:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764076.291581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764076.931661:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764077.315584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764077.955728:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764078.083599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764078.083610:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764078.083614:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764078.083626:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3527 00000100:00080000:3.0:1789764078.083646:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764078.083657:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764078.083699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20e00 x1876700049073664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764078.083729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764078.083733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764078.083734:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764078.083738:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3527 00000100:00080000:3.0:1789764078.083741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764078.083760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764078.083780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764078.083797:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049073664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789764078.083798:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20380 x1876700049073792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764078.083816:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764078.083820:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764078.083822:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764078.083826:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3527 00000100:00080000:3.0:1789764078.083831:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764078.083835:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764078.083840:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764078.083853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20000 x1876700049073920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764078.083862:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764078.083865:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764078.083867:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764078.083873:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764078.083876:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3527 00000100:00080000:3.0:1789764078.083882:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764078.083887:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764078.083892:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764078.083906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21f80 x1876700049074048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764078.083914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764078.083924:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764078.083926:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764078.083933:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3527 00000100:00100000:2.0:1789764078.083935:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764078/real 1789764078] req@ffff9811e1a20e00 x1876700049073664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764078.083939:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764078.083948:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764078.083955:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764078.083962:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20e00 x1876700049073664/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764078.083970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20a80 x1876700049074176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764078.083976:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764078.083984:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764078.083989:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049073664:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764078.084012:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049073792:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764078.084034:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764078.084046:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764078/real 1789764078] req@ffff9811e1a20380 x1876700049073792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764078.084066:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20380 x1876700049073792/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764078.084075:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764078.084078:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764078.084081:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049073792:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764078.084095:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073920:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764078.084116:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764078.084126:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764078/real 1789764078] req@ffff9811e1a20000 x1876700049073920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764078.084138:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20000 x1876700049073920/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764078.084152:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764078.084155:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764078.084158:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049073920:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764078.084164:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074048:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764078.084184:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764078.084195:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764078/real 1789764078] req@ffff9811e1a21f80 x1876700049074048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764078.084206:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21f80 x1876700049074048/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764078.084214:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764078.084217:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764078.084220:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074048:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764078.084235:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074176:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764078.084255:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764078.084266:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764078/real 1789764078] req@ffff9811e1a20a80 x1876700049074176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764078.084292:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20a80 x1876700049074176/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764078.084300:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764078.084303:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764078.084306:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074176:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764078.339596:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764078.979646:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764079.363676:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764080.003708:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764080.387577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764081.027731:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764081.411890:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764082.051675:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764082.437617:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764083.075675:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764083.203547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764083.203560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764083.203565:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764083.203575:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3532 00000100:00080000:3.0:1789764083.203586:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764083.203594:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764083.203634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a20700 x1876700049074304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764083.203668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764083.203673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764083.203675:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764083.203679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3532 00000100:00080000:3.0:1789764083.203697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764083.203701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764083.203711:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764083.203723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21500 x1876700049074432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764083.203789:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049074304:192.168.203.113@tcp:250:kworker.0 00000800:00000100:2.0:1789764083.203851:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764083.203894:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764083/real 1789764083] req@ffff9811e1a20700 x1876700049074304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764083.203912:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a20700 x1876700049074304/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764083.203922:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764083.203926:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764083.203930:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a20700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049074304:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764083.203997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764083.204000:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764083.204003:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764083.204015:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3532 00000100:00080000:3.0:1789764083.204019:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764083.204023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764083.204028:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764083.204040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049074560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764083.204049:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764083.204052:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764083.204054:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764083.204057:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3532 00000100:00080000:3.0:1789764083.204059:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764083.204063:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764083.204067:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764083.204075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049074688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764083.204094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764083.204097:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764083.204099:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764083.204103:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3532 00000100:00080000:3.0:1789764083.204106:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764083.204110:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764083.204115:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764083.204126:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049074816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764083.204391:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049074432:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764083.204415:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764083.204427:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764083/real 1789764083] req@ffff9811e1a21500 x1876700049074432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764083.204440:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21500 x1876700049074432/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764083.204611:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764083.204616:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764083.204621:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049074432:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764083.204627:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074560:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764083.204650:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764083.204662:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764083/real 1789764083] req@ffff9811e1a23480 x1876700049074560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764083.204673:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049074560/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764083.204681:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764083.204685:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764083.204688:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074560:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764083.204694:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074688:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764083.204712:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764083.204721:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764083/real 1789764083] req@ffff9811e1a23800 x1876700049074688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764083.204825:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049074688/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764083.204833:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764083.204837:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764083.204845:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074688:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764083.204852:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074816:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764083.204871:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764083.204880:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764083/real 1789764083] req@ffff9811e1a23100 x1876700049074816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764083.204892:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049074816/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764083.204900:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764083.204903:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764083.204906:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049074816:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764083.459610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764084.099709:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764084.483587:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764085.123673:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764085.507574:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764086.147673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764086.531566:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:0.0:1789764087.171710:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764087.555573:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764088.195647:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764088.323627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764088.323639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764088.323645:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764088.323656:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3537 00000100:00080000:3.0:1789764088.323669:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764088.323677:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764088.323736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049074944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764088.323787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764088.323791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764088.323793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764088.323799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3537 00000100:00080000:3.0:1789764088.323805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764088.323810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764088.323823:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764088.323840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049075072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764088.323851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764088.323855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764088.323857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764088.323862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3537 00000100:00080000:3.0:1789764088.323865:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764088.323869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764088.323874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764088.323891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049075200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764088.323901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764088.323904:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764088.323906:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764088.323910:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3537 00000100:00080000:3.0:1789764088.323920:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764088.323927:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764088.323933:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764088.323946:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049074944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789764088.323947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049075328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764088.324029:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764088.324075:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764088/real 1789764088] req@ffff9811e1a22680 x1876700049074944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764088.324096:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049074944/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764088.324133:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764088.324139:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764088.324143:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049074944:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764088.324151:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049075072:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764088.324183:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764088.324199:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764088/real 1789764088] req@ffff9811e1a21c00 x1876700049075072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764088.324215:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049075072/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764088.324250:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764088.324253:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764088.324258:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00080000:3.0:1789764088.324264:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789764088.324270:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049075072:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764088.324291:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075200:192.168.203.113@tcp:38:kworker.0 00000100:00080000:3.0:1789764088.324312:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764088.324322:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3537 00000800:00000100:0.0:1789764088.324323:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764088.324334:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764088.324341:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764088.324345:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764088/real 1789764088] req@ffff9811e1a21880 x1876700049075200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789764088.324350:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764088.324374:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049075200/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764088.324376:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049075456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764088.324409:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764088.324419:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764088.324424:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075200:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764088.324514:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075328:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764088.324542:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764088.324556:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764088/real 1789764088] req@ffff9811e1a22300 x1876700049075328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764088.324570:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049075328/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764088.324579:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764088.324584:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764088.324590:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075328:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764088.324602:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075456:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764088.324628:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764088.324662:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764088/real 1789764088] req@ffff9811e1a21180 x1876700049075456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764088.324680:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049075456/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764088.324688:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764088.324702:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764088.324706:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075456:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764088.579614:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764089.219872:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764089.603610:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764090.243644:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764090.627738:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764091.267688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764091.651577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764092.291673:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764092.675586:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764093.315615:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764093.443556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764093.443567:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764093.443572:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764093.443582:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3543 00000100:00080000:3.0:1789764093.443612:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764093.443619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764093.443677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22d80 x1876700049075584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764093.443708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764093.443712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764093.443714:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764093.443722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3543 00000100:00080000:3.0:1789764093.443727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764093.443730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764093.443739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764093.443751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22a00 x1876700049075712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764093.443762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764093.443765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764093.443767:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764093.443770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3543 00000100:00080000:3.0:1789764093.443774:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764093.443779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764093.443785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764093.443797:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049075584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789764093.443799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21180 x1876700049075840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764093.443814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764093.443822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764093.443824:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764093.443828:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3543 00000100:00080000:3.0:1789764093.443832:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764093.443837:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764093.443843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764093.443856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22300 x1876700049075968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764093.443866:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:3.0:1789764093.443867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764093.443872:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764093.443874:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764093.443879:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3543 00000100:00080000:3.0:1789764093.443884:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764093.443888:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764093.443893:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764093.443905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21880 x1876700049076096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764093.443919:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764093/real 1789764093] req@ffff9811e1a22d80 x1876700049075584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764093.443944:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22d80 x1876700049075584/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764093.443958:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764093.443961:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764093.443965:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049075584:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764093.443972:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049075712:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764093.443993:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764093.444003:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764093/real 1789764093] req@ffff9811e1a22a00 x1876700049075712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764093.444015:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22a00 x1876700049075712/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764093.444022:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764093.444025:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764093.444028:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049075712:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764093.444056:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075840:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764093.444073:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764093.444082:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764093/real 1789764093] req@ffff9811e1a21180 x1876700049075840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764093.444093:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21180 x1876700049075840/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764093.444110:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764093.444113:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764093.444116:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075840:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764093.444122:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075968:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764093.444138:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764093.444147:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764093/real 1789764093] req@ffff9811e1a22300 x1876700049075968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764093.444167:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22300 x1876700049075968/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764093.444174:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764093.444177:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764093.444180:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049075968:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764093.444185:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076096:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764093.444202:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764093.444210:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764093/real 1789764093] req@ffff9811e1a21880 x1876700049076096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764093.444220:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21880 x1876700049076096/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764093.444227:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764093.444230:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764093.444237:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076096:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764093.699561:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764094.339674:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764094.723547:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764095.363682:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764095.747595:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764096.387652:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764096.771585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764097.411677:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764097.795559:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764098.435597:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764098.563572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764098.563580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764098.563583:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764098.563592:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3548 00000100:00080000:0.0:1789764098.563603:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764098.563610:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764098.563675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049076224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764098.563696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764098.563698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764098.563700:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764098.563703:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3548 00000100:00080000:0.0:1789764098.563706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764098.563708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764098.563716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764098.563725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049076352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764098.563734:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049076224:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789764098.563735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764098.563747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764098.563748:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764098.563751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3548 00000100:00080000:0.0:1789764098.563754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764098.563769:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764098.563773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764098.563782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049076480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764098.563788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764098.563790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764098.563792:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764098.563794:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3548 00000100:00080000:0.0:1789764098.563801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789764098.563803:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789764098.563806:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764098.563810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764098.563818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049076608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764098.563824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764098.563827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764098.563828:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764098.563829:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3548 00000100:00080000:0.0:1789764098.563831:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764098.563834:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764098.563837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764098.563843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049076736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764098.563849:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764098/real 1789764098] req@ffff9811c2e28380 x1876700049076224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764098.563871:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049076224/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764098.563881:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764098.563885:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764098.563889:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049076224:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764098.563911:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049076352:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764098.563929:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764098.563944:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764098/real 1789764098] req@ffff9811c2e28a80 x1876700049076352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764098.563951:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049076352/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764098.563957:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764098.563958:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764098.563960:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049076352:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764098.563963:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076480:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764098.563974:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764098.563980:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764098/real 1789764098] req@ffff9811c2e29f80 x1876700049076480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764098.563986:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049076480/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764098.563990:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764098.563992:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764098.563993:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076480:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764098.563996:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076608:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764098.564005:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764098.564019:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764098/real 1789764098] req@ffff9811c2e2b800 x1876700049076608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764098.564025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049076608/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764098.564034:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764098.564036:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764098.564038:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076608:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764098.564041:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076736:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764098.564054:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764098.564059:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764098/real 1789764098] req@ffff9811c2e2b480 x1876700049076736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764098.564066:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049076736/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764098.564070:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764098.564071:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764098.564073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049076736:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764098.819553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764099.459579:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764099.843563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764100.483615:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764100.867553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764101.507606:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764101.891554:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764102.531627:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764102.915568:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764103.555657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764103.683602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764103.683612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764103.683615:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764103.683624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3553 00000100:00080000:1.0:1789764103.683634:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764103.683640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764103.683682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7480 x1876700049076864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764103.683710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764103.683714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764103.683716:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764103.683720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3553 00000100:00080000:1.0:1789764103.683724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764103.683727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764103.683737:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764103.683748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4000 x1876700049076992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764103.683757:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049076864:192.168.203.113@tcp:250:kworker.0 00000100:00080000:1.0:1789764103.683759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764103.683762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764103.683763:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764103.683783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3553 00000100:00080000:1.0:1789764103.683787:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764103.683790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764103.683794:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764103.683803:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c6680 x1876700049077120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764103.683810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764103.683813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764103.683815:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764103.683818:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3553 00000100:00080000:1.0:1789764103.683820:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789764103.683821:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:1.0:1789764103.683824:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764103.683827:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764103.683834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049077248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764103.683842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764103.683844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764103.683846:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764103.683849:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3553 00000100:00080000:1.0:1789764103.683851:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764103.683854:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764103.683858:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764103.683861:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764103/real 1789764103] req@ffff9811c92c7480 x1876700049076864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764103.683866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049077376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764103.683877:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7480 x1876700049076864/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764103.683886:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764103.683890:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764103.683893:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049076864:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764103.683921:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049076992:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764103.683941:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764103.683949:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764103/real 1789764103] req@ffff9811c92c4000 x1876700049076992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764103.683957:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4000 x1876700049076992/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764103.683962:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764103.683964:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764103.683966:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049076992:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764103.683969:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077120:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764103.683979:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764103.683984:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764103/real 1789764103] req@ffff9811c92c6680 x1876700049077120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764103.683990:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c6680 x1876700049077120/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764103.683994:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764103.683995:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764103.683997:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077120:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764103.684000:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077248:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764103.684008:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764103.684013:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764103/real 1789764103] req@ffff9811c92c5500 x1876700049077248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764103.684025:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049077248/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764103.684030:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764103.684031:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764103.684033:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077248:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764103.684036:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077376:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764103.684046:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764103.684051:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764103/real 1789764103] req@ffff9811c92c4e00 x1876700049077376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764103.684057:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049077376/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764103.684062:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764103.684063:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764103.684065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077376:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764103.939553:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764104.579641:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764104.963560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764105.603688:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764105.987584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764106.627674:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764107.011608:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764107.651702:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764108.035588:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764108.675697:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764108.803630:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764108.803642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764108.803646:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764108.803656:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3558 00000100:00080000:2.0:1789764108.803667:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764108.803675:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764108.803719:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b800 x1876700049077504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764108.803771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764108.803775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764108.803776:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764108.803781:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3558 00000100:00080000:2.0:1789764108.803785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764108.803789:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764108.803799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764108.803814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863a300 x1876700049077632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764108.803825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764108.803828:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764108.803830:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764108.803841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3558 00000100:00080000:2.0:1789764108.803846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764108.803855:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049077504:192.168.203.113@tcp:250:kworker.0 00000100:00080000:2.0:1789764108.803856:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764108.803862:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764108.803873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863b480 x1876700049077760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764108.803902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764108.803906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764108.803907:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764108.803910:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3558 00000100:00080000:2.0:1789764108.803918:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764108.803922:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764108.803927:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764108.803937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d863aa00 x1876700049077888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764108.803943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764108.803946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764108.803947:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764108.803949:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3558 00000100:00080000:2.0:1789764108.803951:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764108.803953:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:2.0:1789764108.803956:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:0.0:1789764108.803964:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764108.803969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811d8638000 x1876700049078016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764108.804024:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764108/real 1789764108] req@ffff9811d863b800 x1876700049077504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764108.804045:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b800 x1876700049077504/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764108.804057:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764108.804061:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764108.804065:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049077504:192.168.203.113@tcp:250:kworker.0 00000100:00100000:0.0:1789764108.804073:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049077632:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764108.804100:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764108.804113:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764108/real 1789764108] req@ffff9811d863a300 x1876700049077632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764108.804126:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863a300 x1876700049077632/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764108.804134:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764108.804137:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764108.804140:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049077632:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764108.804166:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077760:192.168.203.113@tcp:38:kworker.0 00000800:00000100:0.0:1789764108.804186:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764108.804197:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764108/real 1789764108] req@ffff9811d863b480 x1876700049077760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764108.804223:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863b480 x1876700049077760/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764108.804230:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764108.804233:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764108.804236:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077760:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764108.804242:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077888:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764108.804262:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764108.804271:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764108/real 1789764108] req@ffff9811d863aa00 x1876700049077888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764108.804282:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d863aa00 x1876700049077888/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764108.804289:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764108.804292:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764108.804295:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d863aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049077888:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764108.804300:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078016:192.168.203.113@tcp:8:kworker.0 00000800:00000100:0.0:1789764108.804316:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764108.804336:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764108/real 1789764108] req@ffff9811d8638000 x1876700049078016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764108.804349:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811d8638000 x1876700049078016/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764108.804370:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764108.804373:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764108.804376:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811d8638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078016:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764109.059593:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764109.699657:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764110.083648:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764110.723659:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764111.107613:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764111.747642:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764112.131585:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764112.771628:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764113.155622:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764113.795639:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764113.923610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764113.923620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764113.923623:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764113.923632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3563 00000100:00080000:0.0:1789764113.923642:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764113.923649:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764113.923692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29500 x1876700049078144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764113.923722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764113.923727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764113.923728:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764113.923733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3563 00000100:00080000:0.0:1789764113.923738:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764113.923743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764113.923752:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049078144:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789764113.923753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764113.923768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28000 x1876700049078272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764113.923777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764113.923780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764113.923796:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764113.923799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3563 00000100:00080000:0.0:1789764113.923802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764113.923806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764113.923810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764113.923817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a300 x1876700049078400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764113.923825:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789764113.923826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764113.923830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764113.923831:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764113.923834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3563 00000100:00080000:0.0:1789764113.923836:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764113.923838:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764113.923842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764113.923849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b480 x1876700049078528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764113.923854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764113.923856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764113.923857:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764113.923860:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3563 00000100:00080000:0.0:1789764113.923863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764113.923865:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764113.923868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1789764113.923869:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764113/real 1789764113] req@ffff9811c2e29500 x1876700049078144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764113.923884:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b800 x1876700049078656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764113.923890:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29500 x1876700049078144/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764113.923899:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764113.923902:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764113.923905:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049078144:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764113.923932:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049078272:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764113.923948:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764113.923956:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764113/real 1789764113] req@ffff9811c2e28000 x1876700049078272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764113.923964:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28000 x1876700049078272/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764113.923969:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764113.923971:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764113.923973:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049078272:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764113.923976:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078400:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764113.923985:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764113.923990:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764113/real 1789764113] req@ffff9811c2e2a300 x1876700049078400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764113.923997:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a300 x1876700049078400/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764113.924001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764113.924010:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764113.924014:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078400:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764113.924019:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078528:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764113.924038:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764113.924049:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764113/real 1789764113] req@ffff9811c2e2b480 x1876700049078528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764113.924063:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b480 x1876700049078528/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764113.924068:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764113.924070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764113.924072:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078528:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764113.924077:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078656:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764113.924103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764113.924117:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764113/real 1789764113] req@ffff9811c2e2b800 x1876700049078656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764113.924127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b800 x1876700049078656/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764113.924132:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764113.924134:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764113.924136:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049078656:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764114.179577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764114.819650:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764115.203581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764115.843595:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764116.227576:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764116.867592:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764117.251581:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764117.891672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764118.275569:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764118.915611:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764119.043559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764119.043566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764119.043570:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764119.043579:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3568 00000100:00080000:0.0:1789764119.043589:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764119.043595:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764119.043630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29f80 x1876700049078784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764119.043650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764119.043653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764119.043654:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764119.043656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3568 00000100:00080000:0.0:1789764119.043659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764119.043660:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764119.043668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764119.043676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28a80 x1876700049078912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764119.043686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764119.043690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764119.043693:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764119.043695:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049078784:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789764119.043698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3568 00000100:00080000:0.0:1789764119.043706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764119.043724:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764119.043729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764119.043743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28380 x1876700049079040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764119.043749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764119.043752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764119.043753:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764119.043756:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3568 00000100:00080000:0.0:1789764119.043760:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764119.043763:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764119.043766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789764119.043771:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:0.0:1789764119.043775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2aa00 x1876700049079168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764119.043782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764119.043784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764119.043785:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764119.043787:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3568 00000100:00080000:0.0:1789764119.043789:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764119.043791:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764119.043794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764119.043799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2b100 x1876700049079296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764119.043820:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764119/real 1789764119] req@ffff9811c2e29f80 x1876700049078784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764119.043840:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29f80 x1876700049078784/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764119.043864:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764119.043868:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764119.043873:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049078784:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764119.043899:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049078912:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764119.043923:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764119.043933:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764119/real 1789764119] req@ffff9811c2e28a80 x1876700049078912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764119.043942:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28a80 x1876700049078912/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764119.043949:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764119.043952:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764119.043955:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049078912:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764119.043961:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079040:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764119.043980:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764119.043991:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764119/real 1789764119] req@ffff9811c2e28380 x1876700049079040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764119.044000:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28380 x1876700049079040/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764119.044004:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764119.044006:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764119.044008:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079040:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764119.044017:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079168:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764119.044033:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764119.044040:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764119/real 1789764119] req@ffff9811c2e2aa00 x1876700049079168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764119.044059:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2aa00 x1876700049079168/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764119.044067:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764119.044070:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764119.044073:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079168:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764119.044079:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079296:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764119.044103:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764119.044111:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764119/real 1789764119] req@ffff9811c2e2b100 x1876700049079296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764119.044118:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2b100 x1876700049079296/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764119.044122:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764119.044125:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764119.044128:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079296:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764119.299578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764119.939605:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764120.323577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764120.963569:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764121.347560:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764121.987643:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764122.371558:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764123.011744:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764123.395542:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764124.035631:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764124.163585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764124.163593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764124.163597:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764124.163605:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3573 00000100:00080000:1.0:1789764124.163615:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764124.163622:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:1.0:1789764124.163666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c7b80 x1876700049079424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764124.163693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764124.163697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764124.163699:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764124.163714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3573 00000100:00080000:1.0:1789764124.163717:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764124.163720:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764124.163729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764124.163730:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049079424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:1.0:1789764124.163749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5880 x1876700049079552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764124.163758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764124.163761:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764124.163763:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764124.163766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3573 00000100:00080000:1.0:1789764124.163769:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764124.163771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764124.163774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764124.163782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5180 x1876700049079680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764124.163788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764124.163791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764124.163793:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764124.163797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3573 00000100:00080000:1.0:1789764124.163801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764124.163805:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764124.163810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764124.163823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c4e00 x1876700049079808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764124.163839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789764124.163843:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049079552:192.168.203.113@tcp:38:kworker.0 00000100:00080000:1.0:1789764124.163844:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764124.163846:0:36:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764124.163853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3573 00000100:00080000:1.0:1789764124.163857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764124.163859:0:36:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:1.0:1789764124.163863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1789764124.163874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c92c5500 x1876700049079936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764124.163888:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764124.163919:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764124.163943:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764124.419563:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764125.059670:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764125.443550:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764126.083670:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764126.467584:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764127.107635:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789764127.235805:0:2347:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.113:988 00000400:00000100:3.0:1789764127.235866:0:2347:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.113@tcp at host 192.168.203.113 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789764127.235884:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764127.235945:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764127.235955:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764127.235963:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000800:00000100:3.0:1789764127.235978:0:2347:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.113@tcp 00000100:00100000:0.0:1789764127.236132:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764124/real 1789764127] req@ffff9811c92c4e00 x1876700049079808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764127.236158:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c4e00 x1876700049079808/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764127.236171:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764127.236176:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764127.236181:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079808:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764127.236191:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764124/real 1789764127] req@ffff9811c92c5500 x1876700049079936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764127.236208:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5500 x1876700049079936/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764127.236216:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764127.236219:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764127.236223:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079936:192.168.203.113@tcp:8:kworker.0 00000100:00100000:0.0:1789764127.236228:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764124/real 1789764127] req@ffff9811c92c5880 x1876700049079552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764127.236265:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5880 x1876700049079552/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764127.236283:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764127.236287:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764127.236290:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049079552:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764127.236296:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764124/real 1789764127] req@ffff9811c92c5180 x1876700049079680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764127.236309:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c5180 x1876700049079680/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764127.236317:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764127.236320:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764127.236323:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049079680:192.168.203.113@tcp:38:kworker.0 00000100:00100000:0.0:1789764127.236329:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764124/real 1789764127] req@ffff9811c92c7b80 x1876700049079424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764127.236341:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c92c7b80 x1876700049079424/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764127.236349:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764127.236352:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:0.0:1789764127.236355:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c92c7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049079424:192.168.203.113@tcp:250:kworker.0 00000100:00100000:3.0:1789764127.491538:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:1.0:1789764128.131651:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764128.515520:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764129.155672:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764129.283600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764129.283629:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764129.283633:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764129.283643:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3578 00000100:00080000:0.0:1789764129.283653:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764129.283660:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764129.283690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2a680 x1876700049080064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764129.283717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764129.283720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764129.283721:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764129.283724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3578 00000100:00080000:0.0:1789764129.283726:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764129.283728:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764129.283736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764129.283747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2bb80 x1876700049080192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764129.283758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764129.283762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764129.283764:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764129.283767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3578 00000100:00080000:0.0:1789764129.283771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764129.283777:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:2.0:1789764129.283779:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049080064:192.168.203.113@tcp:250:kworker.0 00000008:00100000:0.0:1789764129.283783:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764129.283793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29c00 x1876700049080320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764129.283819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764129.283823:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764129.283825:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764129.283829:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3578 00000100:00080000:0.0:1789764129.283834:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764129.283838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764129.283844:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764129.283857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049080448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764129.283866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764129.283870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764129.283871:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764129.283875:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3578 00000100:00080000:0.0:1789764129.283879:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789764129.283882:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00080000:0.0:1789764129.283886:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764129.283893:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764129.283916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049080576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764129.283941:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764129/real 1789764129] req@ffff9811c2e2a680 x1876700049080064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764129.283963:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2a680 x1876700049080064/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764129.283974:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764129.283978:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764129.283982:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049080064:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764129.283990:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049080192:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764129.284010:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764129.284022:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764129/real 1789764129] req@ffff9811c2e2bb80 x1876700049080192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764129.284034:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2bb80 x1876700049080192/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764129.284041:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764129.284045:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764129.284048:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049080192:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764129.284073:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080320:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764129.284093:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764129.284104:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764129/real 1789764129] req@ffff9811c2e29c00 x1876700049080320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764129.284127:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29c00 x1876700049080320/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764129.284135:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764129.284138:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764129.284141:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080320:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764129.284147:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080448:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764129.284168:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764129.284178:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764129/real 1789764129] req@ffff9811c2e29880 x1876700049080448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764129.284189:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049080448/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764129.284197:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764129.284200:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764129.284203:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080448:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764129.284209:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080576:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764129.284224:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764129.284232:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764129/real 1789764129] req@ffff9811c2e29180 x1876700049080576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764129.284241:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049080576/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764129.284258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764129.284261:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764129.284264:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080576:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764129.539555:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764130.179672:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764130.563556:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764131.203613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764131.587534:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764132.227626:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764132.611591:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764133.251565:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764133.635526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764134.275685:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764134.403626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764134.403638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764134.403642:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764134.403654:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3583 00000100:00080000:3.0:1789764134.403665:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764134.403673:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:3.0:1789764134.403708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a21c00 x1876700049080704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764134.403735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764134.403738:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764134.403757:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764134.403761:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3583 00000100:00080000:3.0:1789764134.403764:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764134.403767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764134.403776:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764134.403785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a22680 x1876700049080832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764134.403790:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049080704:192.168.203.113@tcp:250:kworker.0 00000100:00080000:3.0:1789764134.403796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764134.403799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764134.403800:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764134.403803:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3583 00000100:00080000:3.0:1789764134.403805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764134.403807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764134.403810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764134.403818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23100 x1876700049080960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764134.403824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764134.403826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764134.403828:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764134.403830:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3583 00000100:00080000:3.0:1789764134.403832:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764134.403833:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764134.403836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1789764134.403846:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23800 x1876700049081088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764134.403853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764134.403855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764134.403856:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764134.403858:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3583 00000100:00080000:3.0:1789764134.403860:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764134.403862:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:3.0:1789764134.403865:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000800:00000100:2.0:1789764134.403868:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:3.0:1789764134.403877:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811e1a23480 x1876700049081216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764134.403909:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764134/real 1789764134] req@ffff9811e1a21c00 x1876700049080704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764134.403924:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a21c00 x1876700049080704/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764134.403934:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764134.403938:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764134.403941:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a21c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049080704:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764134.403966:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049080832:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764134.404001:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764134.404013:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764134/real 1789764134] req@ffff9811e1a22680 x1876700049080832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764134.404026:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a22680 x1876700049080832/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764134.404033:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764134.404041:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764134.404043:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a22680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049080832:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764134.404047:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080960:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764134.404062:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764134.404069:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764134/real 1789764134] req@ffff9811e1a23100 x1876700049080960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764134.404077:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23100 x1876700049080960/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764134.404082:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764134.404084:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764134.404086:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049080960:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764134.404089:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081088:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764134.404098:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764134.404103:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764134/real 1789764134] req@ffff9811e1a23800 x1876700049081088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764134.404110:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23800 x1876700049081088/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764134.404114:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764134.404116:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764134.404117:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081088:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764134.404120:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081216:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764134.404134:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764134.404140:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764134/real 1789764134] req@ffff9811e1a23480 x1876700049081216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764134.404147:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811e1a23480 x1876700049081216/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764134.404151:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764134.404153:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764134.404155:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811e1a23480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081216:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764134.659505:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764135.299613:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764135.683526:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764136.323663:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764136.707578:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764137.347617:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764137.731649:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764138.371613:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764138.755582:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:2.0:1789764139.395600:0:5442:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764139.523594:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764139.523603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764139.523607:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764139.523618:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.113@tcp: connect to NID 192.168.203.113@tcp last attempt 3589 00000100:00080000:0.0:1789764139.523647:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.113@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764139.523655:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.113@tcp: import ffff9811e1391000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000100:00100000:0.0:1789764139.523692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28700 x1876700049081344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764139.523713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) c2c87d9a-ec5f-46cf-8e98-545b26703d9f->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764139.523716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764139.523717:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764139.523720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: connect to NID 192.168.203.113@tcp last attempt 3589 00000100:00080000:0.0:1789764139.523722:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764139.523725:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e1a82800: import ffff9811e071a000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764139.523733:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764139.523745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e2ad80 x1876700049081472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764139.523755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764139.523759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764139.523761:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764139.523765:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3589 00000100:00080000:0.0:1789764139.523768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764139.523772:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff9811e0606800: import ffff9811e0717000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764139.523777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683239 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764139.523788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e28e00 x1876700049081600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764139.523798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764139.523806:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049081344:192.168.203.113@tcp:250:kworker.0 00000100:00080000:0.0:1789764139.523808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764139.523814:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764139.523828:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3589 00000100:00080000:0.0:1789764139.523832:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764139.523838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff9811e0606800: import ffff9811c2ed5800 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764139.523843:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764139.523859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29180 x1876700049081728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764139.523868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) cfa8bac1-156b-47a9-afab-fe5753451799->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764139.523872:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764139.523874:0:37:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764139.523877:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: connect to NID 192.168.203.113@tcp last attempt 3589 00000100:00080000:0.0:1789764139.523882:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764139.523886:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff9811e0606800: import ffff9811c2ed2000 using connection 192.168.203.113@tcp/192.168.203.113@tcp 00000008:00100000:0.0:1789764139.523891:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683239 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1789764139.523903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9811c2e29880 x1876700049081856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764139.523913:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764139.523968:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764139/real 1789764139] req@ffff9811c2e28700 x1876700049081344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764139.523988:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28700 x1876700049081344/t0(0) o250->MGC192.168.203.113@tcp@192.168.203.113@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764139.524001:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e1391000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764139.524005:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764139.524009:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:735c3f4e-8e34-4a4b-ad59-3e0fc7e5f9ea:2358:1876700049081344:192.168.203.113@tcp:250:kworker.0 00000100:00100000:2.0:1789764139.524018:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049081472:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764139.524042:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764139.524071:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764139/real 1789764139] req@ffff9811c2e2ad80 x1876700049081472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764139.524085:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e2ad80 x1876700049081472/t0(0) o38->lustre-MDT0000-mdc-ffff9811e1a82800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764139.524094:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e071a000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764139.524097:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764139.524101:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e2ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c2c87d9a-ec5f-46cf-8e98-545b26703d9f:2358:1876700049081472:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764139.524129:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081600:192.168.203.113@tcp:38:kworker.0 00000800:00000100:2.0:1789764139.524151:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764139.524161:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764139/real 1789764139] req@ffff9811c2e28e00 x1876700049081600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764139.524173:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e28e00 x1876700049081600/t0(0) o38->lustre-MDT0000-mdc-ffff9811e0606800@192.168.203.113@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764139.524180:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811e0717000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764139.524183:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764139.524186:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e28e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081600:192.168.203.113@tcp:38:kworker.0 00000100:00100000:2.0:1789764139.524192:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081728:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764139.524220:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764139.524234:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764139/real 1789764139] req@ffff9811c2e29180 x1876700049081728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764139.524245:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29180 x1876700049081728/t0(0) o8->lustre-OST0000-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764139.524258:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764139.524262:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764139.524265:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081728:192.168.203.113@tcp:8:kworker.0 00000100:00100000:2.0:1789764139.524271:0:2358:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081856:192.168.203.113@tcp:8:kworker.0 00000800:00000100:2.0:1789764139.524290:0:2358:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.113@tcp 00000100:00100000:2.0:1789764139.524301:0:2358:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764139/real 1789764139] req@ffff9811c2e29880 x1876700049081856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764139.524313:0:2358:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9811c2e29880 x1876700049081856/t0(0) o8->lustre-OST0001-osc-ffff9811e0606800@192.168.203.113@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764139.524333:0:2358:0:(import.c:65:import_set_state_nolock()) ffff9811c2ed2000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764139.524337:0:2358:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.113@tcp failed (-110) 00000100:00100000:2.0:1789764139.524341:0:2358:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9811c2e29880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:cfa8bac1-156b-47a9-afab-fe5753451799:2358:1876700049081856:192.168.203.113@tcp:8:kworker.0 00000100:00100000:3.0:1789764139.779577:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds 00010000:00010000:3.0:1789764140.419689:0:5443:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789764140.803570:0:7468:0:(client.c:2715:ptlrpc_set_wait()) set ffff9811e135e300 going to sleep for 0 seconds Debug log: 55879 lines, 55879 kept, 0 dropped, 0 bad.